From f04576ac844ed21a1e7d15528ebb3d28849a2695 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sun, 19 Aug 2018 20:42:39 +0800 Subject: [PATCH 01/73] total refactor of P2P network, SPV SDK and interfaces --- interface/p2pclient.go | 27 -- interface/p2pclientimpl.go | 44 --- interface/spvservice.go | 14 +- interface/spvservice_test.go | 11 +- interface/spvserviceimpl.go | 29 +- net/addrmanager.go | 30 +- net/connmanager.go | 45 ++- net/neighbors.go | 72 +++++ net/peer.go | 375 +++++++++++++++++++----- net/peermanager.go | 345 ---------------------- net/peermanager_test.go | 59 ---- net/peers.go | 148 ---------- net/peers_test.go | 49 ---- net/serverpeer.go | 294 +++++++++++++++++++ sdk/p2pclient.go | 27 -- sdk/p2pclient_test.go | 146 ---------- sdk/p2pclientimpl.go | 43 --- sdk/protocal.go | 1 - sdk/spvclient.go | 68 ----- sdk/spvclientimpl.go | 149 ---------- sdk/spvpeer.go | 285 ++++++++++++++++++ sdk/spvservice.go | 30 +- sdk/spvserviceimpl.go | 545 ++++++++++------------------------- sdk/syncmanager.go | 186 ++++++++++++ spvwallet/spvwallet.go | 27 +- 25 files changed, 1427 insertions(+), 1622 deletions(-) delete mode 100644 interface/p2pclient.go delete mode 100644 interface/p2pclientimpl.go create mode 100644 net/neighbors.go delete mode 100644 net/peermanager.go delete mode 100644 net/peermanager_test.go delete mode 100644 net/peers.go delete mode 100644 net/peers_test.go create mode 100644 net/serverpeer.go delete mode 100644 sdk/p2pclient.go delete mode 100644 sdk/p2pclient_test.go delete mode 100644 sdk/p2pclientimpl.go delete mode 100644 sdk/spvclient.go delete mode 100644 sdk/spvclientimpl.go create mode 100644 sdk/spvpeer.go create mode 100644 sdk/syncmanager.go diff --git a/interface/p2pclient.go b/interface/p2pclient.go deleted file mode 100644 index f9fac95..0000000 --- a/interface/p2pclient.go +++ /dev/null @@ -1,27 +0,0 @@ -package _interface - -import ( - "github.com/elastos/Elastos.ELA.SPV/net" -) - -/* -P2P client is the interface to interactive with the peer to peer network implementation, -use this to join the peer to peer network and make communication with other peers. -*/ -type P2PClient interface { - // In this method you will set the peer parameters like clientId, port, services, relay etc. - InitLocalPeer(func(*net.Peer)) - - // Set the message handler - SetMessageHandler(handler func() net.MessageHandler) - - // Start the P2P client - Start() - - // Get the peer manager of this P2P client - PeerManager() *net.PeerManager -} - -func NewP2PClient(magic, maxMsgSize uint32, seeds []string, minOutbound, maxConnections int) P2PClient { - return NewP2PClientImpl(magic, maxMsgSize, seeds, minOutbound, maxConnections) -} diff --git a/interface/p2pclientimpl.go b/interface/p2pclientimpl.go deleted file mode 100644 index 1808ad6..0000000 --- a/interface/p2pclientimpl.go +++ /dev/null @@ -1,44 +0,0 @@ -package _interface - -import ( - "github.com/elastos/Elastos.ELA.SPV/net" -) - -type P2PClientImpl struct { - magic uint32 - maxMsgSize uint32 - - seeds []string - minOutbound int - maxConnections int - pm *net.PeerManager -} - -func NewP2PClientImpl(magic, maxMsgSize uint32, seeds []string, minOutbound, maxConnections int) *P2PClientImpl { - return &P2PClientImpl{ - magic: magic, - maxMsgSize: maxMsgSize, - seeds: seeds, - minOutbound: minOutbound, - maxConnections: maxConnections, - } -} - -func (c *P2PClientImpl) InitLocalPeer(initLocal func(peer *net.Peer)) { - // Create peer manager of the P2P network - local := new(net.Peer) - initLocal(local) - c.pm = net.NewPeerManager(c.magic, c.maxMsgSize, c.seeds, c.minOutbound, c.maxConnections, local) -} - -func (c *P2PClientImpl) SetMessageHandler(messageHandler func() net.MessageHandler) { - c.pm.SetMessageHandler(messageHandler) -} - -func (c *P2PClientImpl) Start() { - c.pm.Start() -} - -func (c *P2PClientImpl) PeerManager() *net.PeerManager { - return c.pm -} diff --git a/interface/spvservice.go b/interface/spvservice.go index b5be784..ee60bd5 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -8,6 +8,16 @@ import ( "github.com/elastos/Elastos.ELA/core" ) +// SPV service config +type SPVServiceConfig struct { + Magic uint32 + Foundation string + ClientId uint64 + Seeds []string + MinOutbound int + MaxConnections int +} + /* SPV service is the interface to interactive with the SPV (Simplified Payment Verification) service implementation running background, you can register specific accounts that you are @@ -74,6 +84,6 @@ type TransactionListener interface { Rollback(height uint32) } -func NewSPVService(magic uint32, foundation string, clientId uint64, seeds []string, minOutbound, maxConnections int) (SPVService, error) { - return NewSPVServiceImpl(magic, foundation, clientId, seeds, minOutbound, maxConnections) +func NewSPVService(config SPVServiceConfig) (SPVService, error) { + return NewSPVServiceImpl(config) } diff --git a/interface/spvservice_test.go b/interface/spvservice_test.go index b6a2f4f..059f543 100644 --- a/interface/spvservice_test.go +++ b/interface/spvservice_test.go @@ -103,7 +103,16 @@ func TestNewSPVService(t *testing.T) { var err error rand.Read(id) binary.Read(bytes.NewReader(id), binary.LittleEndian, clientId) - spv, err = NewSPVService(config.Values().Magic, config.Values().Foundation, clientId, config.Values().SeedList, 8, 100) + + config := SPVServiceConfig{ + ClientId:clientId, + Magic: config.Values().Magic, + Foundation:config.Values().Foundation, + Seeds: config.Values().SeedList, + MinOutbound: 8, + MaxConnections: 100, + } + spv, err = NewSPVService(config) if err != nil { t.Error("NewSPVService error %s", err.Error()) } diff --git a/interface/spvserviceimpl.go b/interface/spvserviceimpl.go index 7e4fead..3b3ad8a 100644 --- a/interface/spvserviceimpl.go +++ b/interface/spvserviceimpl.go @@ -13,7 +13,9 @@ import ( "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/store" + "github.com/elastos/Elastos.ELA.SPV/net" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA/bloom" "github.com/elastos/Elastos.ELA/core" @@ -27,7 +29,7 @@ type SPVServiceImpl struct { listeners map[common.Uint256]TransactionListener } -func NewSPVServiceImpl(magic uint32, foundation string, clientId uint64, seeds []string, minOutbound, maxConnections int) (*SPVServiceImpl, error) { +func NewSPVServiceImpl(config SPVServiceConfig) (*SPVServiceImpl, error) { var err error service := new(SPVServiceImpl) service.headers, err = db.NewHeaderStore() @@ -45,12 +47,27 @@ func NewSPVServiceImpl(magic uint32, foundation string, clientId uint64, seeds [ return nil, err } - spvClient, err := sdk.GetSPVClient(magic, clientId, seeds, minOutbound, maxConnections) - if err != nil { - return nil, err + serverPeerConfig := net.ServerPeerConfig{ + Magic: config.Magic, + Version: p2p.EIP001Version, + PeerId: config.ClientId, + Port: 0, + Seeds: config.Seeds, + MinOutbound: config.MinOutbound, + MaxConnections: config.MaxConnections, + } + + serviceConfig := sdk.SPVServiceConfig{ + Server: net.NewServerPeer(serverPeerConfig), + Foundation: config.Foundation, + HeaderStore: service.headers, + GetFilterData: service.GetFilterData, + CommitTx: service.CommitTx, + OnBlockCommitted: service.OnBlockCommitted, + OnRollback: service.OnRollback, } - service.SPVService, err = sdk.GetSPVService(spvClient, foundation, service.headers, service) + service.SPVService, err = sdk.GetSPVService(serviceConfig) if err != nil { return nil, err } @@ -124,7 +141,7 @@ func (service *SPVServiceImpl) HeaderStore() store.HeaderStore { return service.headers } -func (service *SPVServiceImpl) GetData() ([]*common.Uint168, []*core.OutPoint) { +func (service *SPVServiceImpl) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { ops, err := service.dataStore.Outpoints().GetAll() if err != nil { log.Error("[SPV_SERVICE] GetData error ", err) diff --git a/net/addrmanager.go b/net/addrmanager.go index 5e1e18b..6b695c4 100644 --- a/net/addrmanager.go +++ b/net/addrmanager.go @@ -33,24 +33,24 @@ func newAddressesList() *addrList { } } -func (a *addrList) Put(key string, value *knownAddress) { +func (a *addrList) put(key string, value *knownAddress) { a.list[key] = value } -func (a *addrList) Get(key string) *knownAddress { +func (a *addrList) get(key string) *knownAddress { return a.list[key] } -func (a *addrList) Exit(key string) bool { +func (a *addrList) exist(key string) bool { _, ok := a.list[key] return ok } -func (a *addrList) Del(key string) { +func (a *addrList) del(key string) { delete(a.list, key) } -func (a *addrList) Len() int { +func (a *addrList) size() int { return len(a.list) } @@ -65,7 +65,7 @@ func newAddrManager(minOutbound int) *AddrManager { func (am *AddrManager) NeedMoreAddresses() bool { am.mutex.RLock() defer am.mutex.RUnlock() - return am.addresses.Len() < needAddressThreshold + return am.addresses.size() < needAddressThreshold } func (am *AddrManager) GetOutboundAddresses() []p2p.NetAddress { @@ -76,7 +76,7 @@ func (am *AddrManager) GetOutboundAddresses() []p2p.NetAddress { for _, addr := range SortAddressMap(am.addresses.list) { address := addr.String() // Skip connected address - if am.connected.Exit(address) { + if am.connected.exist(address) { continue } addr.increaseAttempts() @@ -117,10 +117,10 @@ func (am *AddrManager) AddressConnected(na *p2p.NetAddress) { addr := na.String() // Try add to address list am.addOrUpdateAddress(na) - if !am.connected.Exit(addr) { - ka := am.addresses.Get(addr) + if !am.connected.exist(addr) { + ka := am.addresses.get(addr) ka.SaveAddr(na) - am.connected.Put(addr, ka) + am.connected.put(addr, ka) } } @@ -130,10 +130,10 @@ func (am *AddrManager) AddressDisconnect(na *p2p.NetAddress) { addr := na.String() // Update disconnect time - ka := am.addresses.Get(addr) + ka := am.addresses.get(addr) ka.updateLastDisconnect() // Delete from connected list - am.connected.Del(addr) + am.connected.del(addr) } func (am *AddrManager) AddOrUpdateAddress(na *p2p.NetAddress) { @@ -146,12 +146,12 @@ func (am *AddrManager) AddOrUpdateAddress(na *p2p.NetAddress) { func (am *AddrManager) addOrUpdateAddress(na *p2p.NetAddress) { addr := na.String() // Update already known address - ka := am.addresses.Get(addr) + ka := am.addresses.get(addr) if ka == nil { ka := new(knownAddress) ka.SaveAddr(na) // Add to address list - am.addresses.Put(addr, ka) + am.addresses.put(addr, ka) } else { ka.SaveAddr(na) } @@ -161,7 +161,7 @@ func (am *AddrManager) KnowAddresses() []p2p.NetAddress { am.mutex.RLock() defer am.mutex.RUnlock() - nas := make([]p2p.NetAddress, 0, am.addresses.Len()) + nas := make([]p2p.NetAddress, 0, am.addresses.size()) for _, ka := range am.addresses.list { nas = append(nas, ka.NetAddress) } diff --git a/net/connmanager.go b/net/connmanager.go index e79b150..16f42e1 100644 --- a/net/connmanager.go +++ b/net/connmanager.go @@ -10,61 +10,60 @@ import ( ) const ( - ConnTimeOut = 5 - HandshakeTimeout = 3 + DialTimeout = time.Second * 10 + HandshakeTimeout = time.Second * 10 ) -type connMsg struct { - inbound bool - conn net.Conn -} - type ConnectionListener interface { - OnConnection(msg connMsg) } type ConnManager struct { - localPeer *Peer + port uint16 maxConnections int mutex *sync.RWMutex connections map[string]net.Conn - listener ConnectionListener + OnConnection func(conn net.Conn, inbound bool) } -func newConnManager(localPeer *Peer, maxConnections int, listener ConnectionListener) *ConnManager { +func newConnManager(port uint16, maxConnections int) *ConnManager { cm := new(ConnManager) - cm.localPeer = localPeer + cm.port = port cm.maxConnections = maxConnections cm.mutex = new(sync.RWMutex) cm.connections = make(map[string]net.Conn) - cm.listener = listener return cm } -func (cm *ConnManager) ResolveAddr(addr string) (string, error) { +func (cm *ConnManager) resolveAddr(addr string) (string, error) { tcpAddr, err := net.ResolveTCPAddr("tcp", addr) if err != nil { log.Debugf("Can not resolve address %s", addr) return addr, err } - - log.Debugf("Seed %s, resolved addr %s", addr, tcpAddr.String()) return tcpAddr.String(), nil } func (cm *ConnManager) Connect(addr string) { - log.Debugf("Connect addr %s", addr) + tcpAddr, err := cm.resolveAddr(addr) + if err != nil { + return + } + + if cm.IsConnected(tcpAddr) { + log.Debugf("Seed %s already connected", addr) + return + } - conn, err := net.DialTimeout("tcp", addr, time.Second*ConnTimeOut) + conn, err := net.DialTimeout("tcp", tcpAddr, DialTimeout) if err != nil { log.Error("Connect to addr ", addr, " failed, err", err) return } // Callback outbound connection - cm.listener.OnConnection(connMsg{inbound: false, conn: conn}) + cm.OnConnection(conn, false) } func (cm *ConnManager) IsConnected(addr string) bool { @@ -91,7 +90,7 @@ func (cm *ConnManager) PeerDisconnected(addr string) { } func (cm *ConnManager) listenConnection() { - listener, err := net.Listen("tcp", fmt.Sprint(":", cm.localPeer.port)) + listener, err := net.Listen("tcp", fmt.Sprint(":", cm.port)) if err != nil { fmt.Println("Start peer listening err, ", err.Error()) return @@ -104,15 +103,15 @@ func (cm *ConnManager) listenConnection() { fmt.Println("Error accepting ", err.Error()) continue } - log.Debugf("New connection accepted, remote: %s local: %s\n", conn.RemoteAddr(), conn.LocalAddr()) + log.Debugf("New connection accepted, remote: %s local: %s", conn.RemoteAddr(), conn.LocalAddr()) // Callback inbound connection - cm.listener.OnConnection(connMsg{inbound: true, conn: conn}) + cm.OnConnection(conn, true) } } func (cm *ConnManager) monitorConnections() { - ticker := time.NewTicker(time.Second * InfoUpdateDuration) + ticker := time.NewTicker(InfoUpdateDuration) for range ticker.C { cm.mutex.Lock() conns := len(cm.connections) diff --git a/net/neighbors.go b/net/neighbors.go new file mode 100644 index 0000000..fafa7d1 --- /dev/null +++ b/net/neighbors.go @@ -0,0 +1,72 @@ +package net + +import ( + "sync" +) + +type neighbors struct { + sync.Mutex + list map[uint64]*Peer +} + +func (ns *neighbors) Init() { + ns.list = make(map[uint64]*Peer) +} + +func (ns *neighbors) AddNeighbor(peer *Peer) { + ns.Lock() + defer ns.Unlock() + + ns.list[peer.ID()] = peer +} + +func (ns *neighbors) DelNeighbor(id uint64) (*Peer, bool) { + ns.Lock() + defer ns.Unlock() + + peer, ok := ns.list[id] + delete(ns.list, id) + + return peer, ok +} + +func (ns *neighbors) GetNeighborCount() (count int) { + ns.Lock() + defer ns.Unlock() + + for _, n := range ns.list { + if !n.Connected() { + continue + } + count++ + } + return count +} + +func (ns *neighbors) GetNeighborPeers() []*Peer { + ns.Lock() + defer ns.Unlock() + + peers := make([]*Peer, 0, len(ns.list)) + for _, peer := range ns.list { + // Skip disconnected peer + if !peer.Connected() { + continue + } + + peers = append(peers, peer) + } + return peers +} + +func (ns *neighbors) IsNeighborPeer(id uint64) bool { + ns.Lock() + defer ns.Unlock() + + peer, ok := ns.list[id] + if !ok { + return false + } + + return peer.Connected() +} diff --git a/net/peer.go b/net/peer.go index 4770765..cb0c98b 100644 --- a/net/peer.go +++ b/net/peer.go @@ -1,22 +1,43 @@ package net import ( + "container/list" "fmt" + "io" "net" - "time" - "sync/atomic" "strings" + "sync/atomic" + "time" "github.com/elastos/Elastos.ELA.SPV/log" "github.com/elastos/Elastos.ELA.Utility/p2p" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA.Utility/p2p/rw" ) -type PeerHandler struct { - OnDisconnected func(peer *Peer) - MakeMessage func(cmd string) (p2p.Message, error) - HandleMessage func(peer *Peer, msg p2p.Message) error +const ( + // outputBufferSize is the number of elements the output channels use. + outputBufferSize = 50 + + // idleTimeout is the duration of inactivity before we time out a peer. + idleTimeout = 5 * time.Minute +) + +// outMsg is used to house a message to be sent along with a channel to signal +// when the message has been sent (or won't be sent due to things such as +// shutdown) +type outMsg struct { + msg p2p.Message + doneChan chan<- struct{} +} + +type PeerConfig struct { + ProtocolVersion uint32 // The P2P network protocol version + MakeTx func() *msg.Tx + MakeBlock func() *msg.Block + MakeMerkleBlock func() *msg.MerkleBlock + HandleMessage func(peer *Peer, msg p2p.Message) } type Peer struct { @@ -29,16 +50,21 @@ type Peer struct { height uint64 relay uint8 // 1 for true 0 for false - state int32 - conn net.Conn - handler PeerHandler - - msgHelper *p2p.MsgHelper + disconnect int32 + conn net.Conn + + rw rw.MessageRW + handleMessage func(peer *Peer, msg p2p.Message) + outputQueue chan outMsg + sendQueue chan outMsg + sendDoneQueue chan struct{} + inQuit chan struct{} + queueQuit chan struct{} + outQuit chan struct{} + quit chan struct{} } func (p *Peer) String() string { - var state p2p.PeerState - state.SetState(uint(p.state)) return fmt.Sprint( "ID:", p.id, ", Version:", p.version, @@ -47,7 +73,6 @@ func (p *Peer) String() string { ", LastActive:", p.lastActive, ", Height:", p.height, ", Relay:", p.relay, - ", State:", state.String(), ", Addr:", p.Addr().String()) } @@ -103,23 +128,6 @@ func (p *Peer) SetRelay(relay uint8) { p.relay = relay } -func (p *Peer) State() int32 { - return atomic.LoadInt32(&p.state) -} - -func (p *Peer) SetState(state int32) { - atomic.StoreInt32(&p.state, state) -} - -func (p *Peer) Disconnect() { - // Return if peer already disconnected - if p.State() == p2p.INACTIVITY { - return - } - p.SetState(p2p.INACTIVITY) - p.conn.Close() -} - func (p *Peer) SetInfo(msg *msg.Version) { p.id = msg.Nonce p.port = msg.Port @@ -138,48 +146,259 @@ func (p *Peer) Height() uint64 { return p.height } -func (p *Peer) OnError(err error) { - switch err { - case p2p.ErrInvalidHeader, - p2p.ErrUnmatchedMagic, - p2p.ErrMsgSizeExceeded: - log.Error(err) +// Connected returns whether or not the peer is currently connected. +// +// This function is safe for concurrent access. +func (p *Peer) Connected() bool { + return atomic.LoadInt32(&p.disconnect) == 0 +} + +// Disconnect disconnects the peer by closing the connection. Calling this +// function when the peer is already disconnected or in the process of +// disconnecting will have no effect. +func (p *Peer) Disconnect() { + // Return if peer already disconnected + if atomic.AddInt32(&p.disconnect, 1) != 1 { + return + } + + p.conn.Close() + close(p.quit) +} + +func (p *Peer) QuitChan() chan struct{} { + return p.quit +} + +// shouldHandleReadError returns whether or not the passed error, which is +// expected to have come from reading from the remote peer in the inHandler, +// should be logged and responded to with a reject message. +func (p *Peer) shouldHandleReadError(err error) bool { + // No logging or reject message when the peer is being forcibly + // disconnected. + if atomic.LoadInt32(&p.disconnect) != 0 { + return false + } + + // No logging or reject message when the remote peer has been + // disconnected. + if err == io.EOF { + return false + } + if opErr, ok := err.(*net.OpError); ok && !opErr.Temporary() { + return false + } + + return true +} + +func (p *Peer) inHandler() { + // The timer is stopped when a new message is received and reset after it + // is processed. + idleTimer := time.AfterFunc(idleTimeout, func() { + log.Warnf("Peer %s no answer for %s -- disconnecting", p, idleTimeout) p.Disconnect() - case p2p.ErrDisconnected: - p.handler.OnDisconnected(p) - default: - log.Error(err, ", peer id is: ", p.ID()) + }) + +out: + for atomic.LoadInt32(&p.disconnect) == 0 { + // Read a message and stop the idle timer as soon as the read + // is done. The timer is reset below for the next iteration if + // needed. + rmsg, err := p.readMessage() + idleTimer.Stop() + if err != nil { + // Only log the error and send reject message if the + // local peer is not forcibly disconnecting and the + // remote peer has not disconnected. + if p.shouldHandleReadError(err) { + errMsg := fmt.Sprintf("Can't read message from %s: %v", p, err) + if err != io.ErrUnexpectedEOF { + log.Errorf(errMsg) + } + + // Push a reject message for the malformed message and wait for + // the message to be sent before disconnecting. + // + // NOTE: Ideally this would include the command in the header if + // at least that much of the message was valid, but that is not + // currently exposed by wire, so just used malformed for the + // command. + rejectMsg := msg.NewReject("malformed", msg.RejectMalformed, errMsg) + // Send the message and block until it has been sent before returning. + doneChan := make(chan struct{}, 1) + p.QueueMessage(rejectMsg, doneChan) + <-doneChan + } + break out + } + log.Debugf("-----> inHandler [%s] from [0x%x]", rmsg.CMD(), p.id) + + // Handle each message. + p.handleMessage(p, rmsg) + + // A message was received so reset the idle timer. + idleTimer.Reset(idleTimeout) } + + // Ensure the idle timer is stopped to avoid leaking the resource. + idleTimer.Stop() + + // Ensure connection is closed. + p.Disconnect() + + close(p.inQuit) } -func (p *Peer) OnMakeMessage(cmd string) (p2p.Message, error) { - if p.State() == p2p.INACTIVITY { - return nil, fmt.Errorf("-----> [%s] from INACTIVE peer [%d]", cmd, p.id) +func (p *Peer) queueHandler() { + pendingMsgs := list.New() + + // We keep the waiting flag so that we know if we have a message queued + // to the outHandler or not. We could use the presence of a head of + // the list for this but then we have rather racy concerns about whether + // it has gotten it at cleanup time - and thus who sends on the + // message's done channel. To avoid such confusion we keep a different + // flag and pendingMsgs only contains messages that we have not yet + // passed to outHandler. + waiting := false + + // To avoid duplication below. + queuePacket := func(msg outMsg, list *list.List, waiting bool) bool { + if !waiting { + p.sendQueue <- msg + } else { + list.PushBack(msg) + } + // we are always waiting now. + return true } - p.lastActive = time.Now() - return p.handler.MakeMessage(cmd) +out: + for { + select { + case msg := <-p.outputQueue: + waiting = queuePacket(msg, pendingMsgs, waiting) + + // This channel is notified when a message has been sent across + // the network socket. + case <-p.sendDoneQueue: + // No longer waiting if there are no more messages + // in the pending messages queue. + next := pendingMsgs.Front() + if next == nil { + waiting = false + continue + } + + // Notify the outHandler about the next item to + // asynchronously send. + val := pendingMsgs.Remove(next) + p.sendQueue <- val.(outMsg) + + case <-p.quit: + break out + } + } + + // Drain any wait channels before we go away so we don't leave something + // waiting for us. + for e := pendingMsgs.Front(); e != nil; e = pendingMsgs.Front() { + val := pendingMsgs.Remove(e) + msg := val.(outMsg) + if msg.doneChan != nil { + msg.doneChan <- struct{}{} + } + } +cleanup: + for { + select { + case msg := <-p.outputQueue: + if msg.doneChan != nil { + msg.doneChan <- struct{}{} + } + default: + break cleanup + } + } + close(p.queueQuit) + log.Tracef("Peer queue handler done for %s", p) } -func (p *Peer) OnMessageDecoded(message p2p.Message) { - log.Debugf("-----> [%s] from peer [%d] STARTED", message.CMD(), p.id) - if err := p.handler.HandleMessage(p, message); err != nil { - log.Error(err) +func (p *Peer) outHandler() { +out: + for { + select { + case msg := <-p.sendQueue: + err := p.writeMessage(msg.msg) + if err != nil { + p.Disconnect() + if msg.doneChan != nil { + msg.doneChan <- struct{}{} + } + continue + } + log.Debugf("-----> outHandler [%s] to [0x%x]", msg.msg.CMD(), p.id) + + if msg.doneChan != nil { + msg.doneChan <- struct{}{} + } + p.sendDoneQueue <- struct{}{} + + case <-p.quit: + break out + } + } + + <-p.queueQuit + + // Drain any wait channels before we go away so we don't leave something + // waiting for us. We have waited on queueQuit and thus we can be sure + // that we will not miss anything sent on sendQueue. +cleanup: + for { + select { + case msg := <-p.sendQueue: + if msg.doneChan != nil { + msg.doneChan <- struct{}{} + } + // no need to send on sendDoneQueue since queueHandler + // has been waited on and already exited. + default: + break cleanup + } } - log.Debugf("-----> [%s] from peer [%d] FINISHED", message.CMD(), p.id) + close(p.outQuit) + log.Tracef("Peer output handler done for %s", p) } -func (p *Peer) Start() { - p.msgHelper.Read() +func (p *Peer) readMessage() (p2p.Message, error) { + return p.rw.ReadMessage(p.conn) +} + +func (p *Peer) writeMessage(msg p2p.Message) error { + // Don't do anything if we're disconnecting. + if atomic.LoadInt32(&p.disconnect) != 0 { + return nil + } + + return p.rw.WriteMessage(p.conn, msg) } -func (p *Peer) Send(msg p2p.Message) { - if p.State() == p2p.INACTIVITY { - log.Errorf("-----> Push [%s] to INACTIVE peer [%d]", msg.CMD(), p.id) +func (p *Peer) QueueMessage(msg p2p.Message, doneChan chan<- struct{}) { + if atomic.LoadInt32(&p.disconnect) != 0 { + if doneChan != nil { + go func() { + doneChan <- struct{}{} + }() + } return } - log.Debugf("-----> Push [%s] to peer [%d] STARTED", msg.CMD(), p.id) - p.msgHelper.Write(msg) - log.Debugf("-----> Push [%s] to peer [%d] FINISHED", msg.CMD(), p.id) + p.outputQueue <- outMsg{msg: msg, doneChan: doneChan} +} + +func (p *Peer) start() { + go p.inHandler() + go p.queueHandler() + go p.outHandler() } func (p *Peer) NewVersionMsg() *msg.Version { @@ -194,16 +413,38 @@ func (p *Peer) NewVersionMsg() *msg.Version { return version } -func (p *Peer) SetPeerHandler(handler PeerHandler) { - p.handler = handler +func (p *Peer) SetConfig(config PeerConfig) { + rwConfig := rw.MessageConfig{ + ProtocolVersion: config.ProtocolVersion, + MakeTx: config.MakeTx, + MakeBlock: config.MakeBlock, + MakeMerkleBlock: config.MakeMerkleBlock, + } + p.rw.SetConfig(rwConfig) + + // Upgrade peer message handler + previousHandler := p.handleMessage + p.handleMessage = func(peer *Peer, msg p2p.Message) { + previousHandler(peer, msg) + config.HandleMessage(peer, msg) + } } -func NewPeer(magic, maxMsgSize uint32, conn net.Conn) *Peer { - peer := new(Peer) - peer.conn = conn - copy(peer.ip16[:], getIp(conn)) - peer.msgHelper = p2p.NewMsgHelper(magic, maxMsgSize, conn, peer) - return peer +func NewPeer(magic uint32, conn net.Conn) *Peer { + p := Peer{ + conn: conn, + rw: rw.GetMesssageRW(magic), + outputQueue: make(chan outMsg, outputBufferSize), + sendQueue: make(chan outMsg, 1), // nonblocking sync + sendDoneQueue: make(chan struct{}, 1), // nonblocking sync + inQuit: make(chan struct{}), + queueQuit: make(chan struct{}), + outQuit: make(chan struct{}), + quit: make(chan struct{}), + } + + copy(p.ip16[:], getIp(conn)) + return &p } func getIp(conn net.Conn) []byte { diff --git a/net/peermanager.go b/net/peermanager.go deleted file mode 100644 index fe4d158..0000000 --- a/net/peermanager.go +++ /dev/null @@ -1,345 +0,0 @@ -package net - -import ( - "errors" - "time" - "fmt" - - "github.com/elastos/Elastos.ELA.SPV/log" - - "github.com/elastos/Elastos.ELA.Utility/p2p" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" -) - -const ( - MinConnections = 3 - InfoUpdateDuration = 5 - KeepAliveTimeout = 30 -) - -type peerMsg struct { - peer *Peer - inbound bool - connDone chan struct{} -} - -// Handle the message creation, allocation etc. -type MessageHandler interface { - // A handshake message received - OnHandshake(v *msg.Version) error - - // Create a message instance by the given cmd parameter - MakeMessage(cmd string) (p2p.Message, error) - - // VerAck message received from a connected peer - // which means the connected peer is established - OnPeerEstablish(*Peer) - - // Handle messages received from the connected peer - HandleMessage(*Peer, p2p.Message) error -} - -type PeerManager struct { - magic uint32 - maxMsgSize uint32 - seeds []string - connectionQueue chan peerMsg - disconnectQueue chan *Peer - - *Peers - handler func() MessageHandler - am *AddrManager - cm *ConnManager -} - -func NewPeerManager(magic, maxMsgSize uint32, seeds []string, minOutbound, maxConnections int, localPeer *Peer) *PeerManager { - // Initiate PeerManager - pm := new(PeerManager) - pm.magic = magic - pm.maxMsgSize = maxMsgSize - pm.seeds = seeds - pm.connectionQueue = make(chan peerMsg, 1) - pm.disconnectQueue = make(chan *Peer, 1) - pm.Peers = newPeers(localPeer) - pm.am = newAddrManager(minOutbound) - pm.cm = newConnManager(localPeer, maxConnections, pm) - return pm -} - -func (pm *PeerManager) SetMessageHandler(messageHandler func() MessageHandler) { - pm.handler = messageHandler -} - -func (pm *PeerManager) Start() { - log.Info("PeerManager start") - go pm.peersHandler() - go pm.keepConnections() - go pm.am.monitorAddresses() - go pm.cm.listenConnection() - go pm.cm.monitorConnections() -} - -func (pm *PeerManager) OnConnection(msg connMsg) { - // Create peer connection message - doneChan := make(chan struct{}) - pm.connectionQueue <- peerMsg{ - connDone: doneChan, - peer: NewPeer(pm.magic, pm.maxMsgSize, msg.conn), - } - <-doneChan -} - -// peersHandler handle peers from inbound/outbound and disconnected peers. -// This method will help to finish peers handshake progress and quit progress. -func (pm *PeerManager) peersHandler() { - for { - select { - case msg := <-pm.connectionQueue: - // Peers come from this queue are connected peers - pm.handshake(msg.peer, msg.inbound, msg.connDone) - - case peer := <-pm.disconnectQueue: - // Peers come from this queue are disconnected - pm.PeerDisconnected(peer) - } - } -} - -func (pm *PeerManager) handshake(peer *Peer, inbound bool, connDone chan struct{}) { - // Create message handler instance - handler := pm.handler() - - // doneChan notify to finish current handshake - doneChan := make(chan struct{}) - - // Set handshake handler - peer.SetPeerHandler(PeerHandler{ - // New peer can only send handshake messages - MakeMessage: func(cmd string) (p2p.Message, error) { - switch cmd { - case p2p.CmdVersion: - return new(msg.Version), nil - case p2p.CmdVerAck: - return new(msg.VerAck), nil - default: - peer.Disconnect() - return nil, fmt.Errorf("none handshake message [%s] received from new peer", cmd) - } - }, - - // Handle handshake messages - HandleMessage: func(peer *Peer, message p2p.Message) error { - switch m := message.(type) { - case *msg.Version: - // Peer not in handshake state - if peer.State() != p2p.INIT && peer.State() != p2p.HAND { - peer.Disconnect() - return fmt.Errorf("peer handshake with unknown state %d", peer.State()) - } - // Callback handshake message first - if err := handler.OnHandshake(m); err != nil { - peer.Disconnect() - return err - } - - // Check if handshake with itself - if m.Nonce == pm.Local().ID() { - peer.Disconnect() - return errors.New("Peer handshake with itself") - } - - // If peer already connected, disconnect previous peer - if oldPeer, ok := pm.RemovePeer(m.Nonce); ok { - log.Warnf("Peer %d reconnect", m.Nonce) - oldPeer.Disconnect() - } - - // Set peer info with version message - peer.SetInfo(m) - - if inbound { - // Replay inbound handshake - peer.SetState(p2p.HANDSHAKE) - peer.Send(pm.Local().NewVersionMsg()) - - } else { - // Finish outbound handshake - peer.SetState(p2p.HANDSHAKED) - peer.Send(new(msg.VerAck)) - } - - case *msg.VerAck: - // Peer not in handshake state - if peer.State() != p2p.HANDSHAKE && peer.State() != p2p.HANDSHAKED { - peer.Disconnect() - return fmt.Errorf("peer handshake with unknown state %d", peer.State()) - } - - // Finish inbound handshake - if inbound { - peer.Send(new(msg.VerAck)) - } - - // Mark peer as establish - peer.SetState(p2p.ESTABLISH) - - // Notify peer establish - handler.OnPeerEstablish(peer) - - // Update peer's message handler - peer.SetPeerHandler(pm.NewPeerHandler(handler)) - - // Get more addresses - if pm.am.NeedMoreAddresses() { - peer.Send(new(msg.GetAddr)) - } - - // Notify handshake finished - doneChan <- struct{}{} - } - return nil - }, - - // Peer disconnected - OnDisconnected: func(peer *Peer) { - pm.disconnectQueue <- peer - }, - }) - - // Start protocol - if inbound { - // Start inbound handshake - peer.SetState(p2p.INIT) - } else { - // Start outbound handshake - peer.SetState(p2p.HAND) - peer.Send(pm.Local().NewVersionMsg()) - } - - peer.Start() - - // Wait for handshake progress finish or timeout - timer := time.NewTimer(time.Second * HandshakeTimeout) - - select { - case <-doneChan: - // Stop timeout timer - timer.Stop() - - // Add peer to neighbor list - pm.PeerConnected(peer) - - case <-timer.C: - // Disconnect peer for handshake timeout - peer.Disconnect() - } - - // Release connection - connDone <- struct{}{} -} - -func (pm *PeerManager) PeerConnected(peer *Peer) { - log.Trace("PeerManager add connected peer:", peer) - // Add peer to list - pm.Peers.AddPeer(peer) - // Mark addr as connected - addr := peer.Addr() - pm.am.AddressConnected(addr) - pm.cm.PeerConnected(addr.String(), peer.conn) -} - -func (pm *PeerManager) PeerDisconnected(peer *Peer) { - if peer, ok := pm.Peers.RemovePeer(peer.ID()); ok { - log.Trace("PeerManager peer disconnected:", peer) - na := peer.Addr() - addr := na.String() - pm.am.AddressDisconnect(na) - pm.cm.PeerDisconnected(addr) - peer.Disconnect() - } -} - -func (pm *PeerManager) KnownAddresses() []p2p.NetAddress { - return pm.am.KnowAddresses() -} - -func (pm *PeerManager) keepConnections() { - for { - // connect seeds first - if pm.PeersCount() < MinConnections { - for _, seed := range pm.seeds { - addr, err := pm.cm.ResolveAddr(seed) - if err != nil { - continue - } - - if pm.cm.IsConnected(addr) { - log.Debugf("Seed %s already connected", seed) - continue - } - pm.cm.Connect(addr) - } - - } else if pm.PeersCount() < pm.am.minOutbound { - for _, addr := range pm.am.GetOutboundAddresses() { - pm.cm.Connect(addr.String()) - } - } - - // request more addresses - if pm.am.NeedMoreAddresses() { - pm.Broadcast(new(msg.GetAddr)) - } - - time.Sleep(time.Second * InfoUpdateDuration) - } -} - -func (pm *PeerManager) NewPeerHandler(handler MessageHandler) PeerHandler { - return PeerHandler{ - MakeMessage: func(cmd string) (p2p.Message, error) { - switch cmd { - case p2p.CmdGetAddr: - return new(msg.GetAddr), nil - case p2p.CmdAddr: - return new(msg.Addr), nil - default: - return handler.MakeMessage(cmd) - } - }, - - HandleMessage: func(peer *Peer, message p2p.Message) error { - switch m := message.(type) { - case *msg.GetAddr: - peer.Send(msg.NewAddr(pm.am.RandGetAddresses())) - - case *msg.Addr: - for _, addr := range m.AddrList { - // Skip local peer - if addr.ID == pm.Local().ID() { - continue - } - // Skip peer already connected - if pm.EstablishedPeer(addr.ID) { - continue - } - // Skip invalid port - if addr.Port == 0 { - continue - } - // Save to address list - pm.am.AddOrUpdateAddress(&addr) - } - - default: - return handler.HandleMessage(peer, m) - } - - return nil - }, - - OnDisconnected: func(peer *Peer) { - pm.disconnectQueue <- peer - }, - } -} diff --git a/net/peermanager_test.go b/net/peermanager_test.go deleted file mode 100644 index ee2ba11..0000000 --- a/net/peermanager_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package net - -import ( - "math/rand" - "net" - "testing" - - "github.com/elastos/Elastos.ELA.SPV/log" - - "github.com/elastos/Elastos.ELA.Utility/p2p" - "github.com/stretchr/testify/assert" -) - -func TestPeerManager_AddConnectedPeer(t *testing.T) { - log.Init(0, 5, 10) - pm := NewPeerManager(123456, 1024, nil, 4, 100, nil) - addPeer := func() { - ADD: - peer := new(Peer) - peer.conn = new(net.TCPConn) - peer.id = rand.Uint64() - peer.height = rand.Uint64() - peer.SetState(p2p.ESTABLISH) - rand.Read(peer.ip16[:]) - peer.port = uint16(rand.Uint32()) - pm.PeerConnected(peer) - assert.Equal(t, true, pm.cm.IsConnected(peer.Addr().String())) - if pm.PeersCount() < 10 { - goto ADD - } - } - - done := make(chan struct{}) - - go func() { - for { - addPeer() - } - }() - - go func() { - count := 0 - for { - NEXT: - peer := pm.GetSyncPeer() - if peer == nil { - goto NEXT - } - pm.PeerDisconnected(peer) - assert.Equal(t, false, pm.cm.IsConnected(peer.Addr().String())) - count++ - if count > 100 { - done <- struct{}{} - } - } - }() - - <-done -} diff --git a/net/peers.go b/net/peers.go deleted file mode 100644 index 5e0efd0..0000000 --- a/net/peers.go +++ /dev/null @@ -1,148 +0,0 @@ -package net - -import ( - "sync" - - . "github.com/elastos/Elastos.ELA.Utility/p2p" -) - -type Peers struct { - syncPeer *Peer - - local *Peer - peersLock *sync.RWMutex - peers map[uint64]*Peer -} - -func newPeers(localPeer *Peer) *Peers { - peers := new(Peers) - peers.local = localPeer - peers.peersLock = new(sync.RWMutex) - peers.peers = make(map[uint64]*Peer) - return peers -} - -func (p *Peers) Local() *Peer { - return p.local -} - -func (p *Peers) AddPeer(peer *Peer) { - p.peersLock.Lock() - defer p.peersLock.Unlock() - - p.peers[peer.ID()] = peer -} - -func (p *Peers) Exist(peer *Peer) bool { - p.peersLock.RLock() - defer p.peersLock.RUnlock() - - _, ok := p.peers[peer.ID()] - return ok -} - -func (p *Peers) RemovePeer(id uint64) (*Peer, bool) { - p.peersLock.Lock() - defer p.peersLock.Unlock() - - if p.syncPeer != nil && id == p.syncPeer.ID() { - p.syncPeer = nil - } - - peer, ok := p.peers[id] - delete(p.peers, id) - - return peer, ok -} - -func (p *Peers) PeersCount() int { - p.peersLock.RLock() - defer p.peersLock.RUnlock() - - return len(p.peers) -} - -func (p *Peers) ConnectedPeers() []*Peer { - p.peersLock.RLock() - defer p.peersLock.RUnlock() - - peers := make([]*Peer, 0, len(p.peers)) - for _, v := range p.peers { - peers = append(peers, v) - } - return peers -} - -func (p *Peers) EstablishedPeer(id uint64) bool { - p.peersLock.RLock() - defer p.peersLock.RUnlock() - - peer, ok := p.peers[id] - if !ok { - return false - } - - return peer.State() == ESTABLISH -} - -func (p *Peers) GetBestPeer() *Peer { - p.peersLock.RLock() - defer p.peersLock.RUnlock() - - var bestPeer *Peer - for _, peer := range p.peers { - - // Skip unestablished peer - if peer.State() != ESTABLISH { - continue - } - - if bestPeer == nil { - bestPeer = peer - continue - } - - if peer.height > bestPeer.height { - bestPeer = peer - } - } - - return bestPeer -} - -func (p *Peers) Broadcast(msg Message) { - // Make a copy of neighbor peers list, - // This can prevent mutex lock when peer.Send() - // method fire a disconnect event. - neighbors := p.ConnectedPeers() - - // Do broadcast - go func() { - for _, peer := range neighbors { - - // Skip unestablished peer - if peer.State() != ESTABLISH { - continue - } - - // Skip non relay peer - if peer.Relay() == 0 { - continue - } - - peer.Send(msg) - } - }() -} - -func (p *Peers) ClearSyncPeer() { - p.syncPeer = nil -} - -func (p *Peers) GetSyncPeer() *Peer { - if p.syncPeer == nil { - p.syncPeer = p.GetBestPeer() - } - - return p.syncPeer -} diff --git a/net/peers_test.go b/net/peers_test.go deleted file mode 100644 index 2cd51a7..0000000 --- a/net/peers_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package net - -import ( - "math/rand" - "testing" - "github.com/elastos/Elastos.ELA.Utility/p2p" -) - -func TestPeers_GetSyncPeer(t *testing.T) { - peers := newPeers(new(Peer)) - addPeer := func() { - ADD: - peer := new(Peer) - peer.id = rand.Uint64() - peer.height = rand.Uint64() - peer.SetState(p2p.ESTABLISH) - peers.AddPeer(peer) - if peers.PeersCount() < 10 { - goto ADD - } - } - - done := make(chan struct{}) - - go func() { - for { - addPeer() - } - }() - - go func() { - count := 0 - for { - NEXT: - peer := peers.GetSyncPeer() - if peer == nil { - goto NEXT - } - t.Logf("peers.GetSyncPeer() %v", peer) - peers.RemovePeer(peer.ID()) - count++ - if count > 100 { - done <- struct{}{} - } - } - }() - - <-done -} diff --git a/net/serverpeer.go b/net/serverpeer.go new file mode 100644 index 0000000..e9b9317 --- /dev/null +++ b/net/serverpeer.go @@ -0,0 +1,294 @@ +package net + +import ( + "errors" + "github.com/elastos/Elastos.ELA.SPV/log" + "time" + + "github.com/elastos/Elastos.ELA.Utility/p2p" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "net" +) + +const ( + MinConnections = 3 + InfoUpdateDuration = time.Second * 5 +) + +// P2P network config +type ServerPeerConfig struct { + Magic uint32 + Version uint32 + PeerId uint64 + Port uint16 + Seeds []string + MinOutbound int + MaxConnections int +} + +// Handle the message creation, allocation etc. +type PeerManageConfig struct { + // A handshake message received + OnHandshake func(v *msg.Version) error + + // VerAck message received from a connected peer + // which means the connected peer is established + OnPeerEstablish func(*Peer) +} + +type ServerPeer struct { + Peer + magic uint32 + seeds []string + neighbors + quitChan chan uint64 + am *AddrManager + cm *ConnManager + config PeerManageConfig +} + +func NewServerPeer(config ServerPeerConfig) *ServerPeer { + // Initiate ServerPeer + sp := new(ServerPeer) + sp.id = config.PeerId + sp.version = config.Version + sp.port = config.Port + sp.magic = config.Magic + sp.seeds = config.Seeds + sp.neighbors.Init() + sp.quitChan = make(chan uint64, 1) + sp.am = newAddrManager(config.MinOutbound) + sp.cm = newConnManager(sp.port, config.MaxConnections) + sp.cm.OnConnection = sp.OnConnection + return sp +} + +func (sp *ServerPeer) SetConfig(config PeerManageConfig) { + sp.config = config +} + +func (sp *ServerPeer) Start() { + log.Info("ServerPeer start") + go sp.keepConnections() + go sp.peerQuitHandler() + go sp.am.monitorAddresses() + go sp.cm.listenConnection() + go sp.cm.monitorConnections() +} + +func (sp *ServerPeer) OnConnection(conn net.Conn, inbound bool) { + // Start handshake + doneChan := make(chan struct{}) + go func() { + sp.handshake(NewPeer(sp.magic, conn), inbound, doneChan) + }() + <-doneChan +} + +func (sp *ServerPeer) readVersionMsg(peer *Peer) error { + message, err := peer.readMessage() + if err != nil { + return err + } + + version, ok := message.(*msg.Version) + if !ok { + errMsg := "A version message must precede all others" + log.Error(errMsg) + + reject := msg.NewReject(message.CMD(), msg.RejectMalformed, errMsg) + return peer.writeMessage(reject) + } + + if err := sp.config.OnHandshake(version); err != nil { + return err + } + + // Check if handshake with itself + if version.Nonce == sp.ID() { + peer.Disconnect() + return errors.New("peer handshake with itself") + } + + // If peer already connected, disconnect previous peer + if oldPeer, ok := sp.DelNeighbor(version.Nonce); ok { + log.Warnf("Peer %d reconnect", version.Nonce) + oldPeer.Disconnect() + } + + // Set peer info with version message + peer.SetInfo(version) + + return nil +} + +func (sp *ServerPeer) inboundProtocol(peer *Peer) error { + if err := sp.readVersionMsg(peer); err != nil { + return err + } + + return peer.writeMessage(sp.NewVersionMsg()) +} + +func (sp *ServerPeer) outboundProtocol(peer *Peer) error { + if err := peer.writeMessage(sp.NewVersionMsg()); err != nil { + return err + } + + return sp.readVersionMsg(peer) +} + +func (sp *ServerPeer) handshake(peer *Peer, inbound bool, doneChan chan struct{}) { + errChan := make(chan error) + go func() { + if inbound { + errChan <- sp.inboundProtocol(peer) + } else { + errChan <- sp.outboundProtocol(peer) + } + }() + + select { + case err := <-errChan: + if err != nil { + return + } + + case <-time.After(HandshakeTimeout): + // Disconnect peer for handshake timeout + peer.Disconnect() + + // Notify handshake done + doneChan <- struct{}{} + return + } + + // Wait for peer quit + go func() { + select { + case <-peer.quit: + sp.quitChan <- peer.id + } + }() + + // Add peer to neighbor list + sp.AddToNeighbors(peer) + + // Notify handshake done + doneChan <- struct{}{} + + // Update peer's message config + peer.handleMessage = sp.baseMessageHandler() + + // Start peer + peer.start() + + // Send our verack message now that the IO processing machinery has started. + peer.QueueMessage(new(msg.VerAck), nil) +} + +func (sp *ServerPeer) peerQuitHandler() { + for peerId := range sp.quitChan { + if peer, ok := sp.neighbors.DelNeighbor(peerId); ok { + log.Trace("ServerPeer peer disconnected:", peer) + na := peer.Addr() + addr := na.String() + sp.am.AddressDisconnect(na) + sp.cm.PeerDisconnected(addr) + } + } +} + +func (sp *ServerPeer) AddToNeighbors(peer *Peer) { + log.Trace("ServerPeer add connected peer:", peer) + // Add peer to list + sp.AddNeighbor(peer) + + // Mark addr as connected + addr := peer.Addr() + sp.am.AddressConnected(addr) + sp.cm.PeerConnected(addr.String(), peer.conn) +} + +func (sp *ServerPeer) KnownAddresses() []p2p.NetAddress { + return sp.am.KnowAddresses() +} + +func (sp *ServerPeer) keepConnections() { + for { + // connect seeds first + if sp.GetNeighborCount() < MinConnections { + for _, seed := range sp.seeds { + sp.cm.Connect(seed) + } + + } else if sp.GetNeighborCount() < sp.am.minOutbound { + for _, addr := range sp.am.GetOutboundAddresses() { + sp.cm.Connect(addr.String()) + } + } + + // request more addresses + if sp.am.NeedMoreAddresses() { + sp.Broadcast(new(msg.GetAddr)) + } + + time.Sleep(InfoUpdateDuration) + } +} + +func (sp *ServerPeer) baseMessageHandler() func(peer *Peer, message p2p.Message) { + return func(peer *Peer, message p2p.Message) { + switch m := message.(type) { + case *msg.VerAck: + // Notify peer establish + sp.config.OnPeerEstablish(peer) + + case *msg.GetAddr: + peer.QueueMessage(msg.NewAddr(sp.am.RandGetAddresses()), nil) + + case *msg.Addr: + for _, addr := range m.AddrList { + // Skip local peer + if addr.ID == sp.ID() { + continue + } + // Skip peer already connected + if sp.IsNeighborPeer(addr.ID) { + continue + } + // Skip invalid port + if addr.Port == 0 { + continue + } + // Save to address list + sp.am.AddOrUpdateAddress(&addr) + } + } + } +} + +func (sp *ServerPeer) Broadcast(msg p2p.Message) { + // Make a copy of neighbor peers list, + // This can prevent mutex lock when peer.Send() + // method fire a disconnect event. + neighbors := sp.GetNeighborPeers() + + // Do broadcast + go func() { + for _, peer := range neighbors { + + // Skip disconnected peer + if !peer.Connected() { + continue + } + + // Skip non relay peer + if peer.Relay() == 0 { + continue + } + + peer.QueueMessage(msg, nil) + } + }() +} diff --git a/sdk/p2pclient.go b/sdk/p2pclient.go deleted file mode 100644 index 65ca2d1..0000000 --- a/sdk/p2pclient.go +++ /dev/null @@ -1,27 +0,0 @@ -package sdk - -import ( - "github.com/elastos/Elastos.ELA.SPV/net" -) - -/* -P2P client is the interface to interactive with the peer to peer network, -use this to join the peer to peer network and make communication with other peers. -*/ -type P2PClient interface { - // Start the P2P client - Start() - - // Get the peer manager of this P2P client - PeerManager() *net.PeerManager -} - -// To get a P2P client, you need to set a magic number and a client ID to identify this peer in the peer to peer network. -// Magic number is the peer to peer network id for the peers in the same network to identify each other, -// MaxMsgSize is the max size of a message in this P2P network. -// and client id is the unique id to identify the current peer in this peer to peer network. -// seeds is a list which is the other peers IP:Port addresses. -// port is the port number for this client listening inbound connections. -func GetP2PClient(magic, maxMsgSize uint32, clientId uint64, seeds []string, port uint16, minOutbound, maxConnections int) (P2PClient, error) { - return NewP2PClientImpl(magic, maxMsgSize, clientId, seeds, port, minOutbound, maxConnections) -} diff --git a/sdk/p2pclient_test.go b/sdk/p2pclient_test.go deleted file mode 100644 index d71809f..0000000 --- a/sdk/p2pclient_test.go +++ /dev/null @@ -1,146 +0,0 @@ -package sdk - -import ( - "fmt" - "math/rand" - "sort" - "testing" - "time" - - "github.com/elastos/Elastos.ELA.SPV/log" - "github.com/elastos/Elastos.ELA.SPV/net" - - "github.com/elastos/Elastos.ELA.Utility/p2p" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" -) - -func TestP2PClient(t *testing.T) { - log.Init(0, 5, 100) - - var clientNumber = 150 - - var clients []*Client - for i := 0; i < clientNumber; i++ { - client, err := NewClient(i, randomSeeds(clientNumber)) - if err != nil { - t.Errorf("GetP2PClient failed %s", err.Error()) - } - clients = append(clients, client) - client.Start() - } - - go monitorConnections(clients) - select {} -} - -func randomSeeds(num int) []string { - seeds := make([]string, 0, num) - for i := 0; i < 5; i++ { - seeds = append(seeds, fmt.Sprint("127.0.0.1:", 50000+rand.Intn(num))) - } - return seeds -} - -func monitorConnections(clients []*Client) { - ticker := time.NewTicker(time.Second * 5) - num := len(clients) - conns := make([]int, num) - addrs := make([]int, num) - for range ticker.C { - var totalAddr = 0 - var totalConn = 0 - for i, c := range clients { - addrs[i] = len(c.PeerManager().KnownAddresses()) - conns[i] = c.PeerManager().PeersCount() - totalAddr += addrs[i] - totalConn += conns[i] - } - sort.Ints(conns) - sort.Ints(addrs) - avgAddr := totalAddr / num - avgConn := totalConn / num - fmt.Printf("[Addresses] Min: %d, Max: %d, Avg: %d\n", addrs[0], addrs[num-1], avgAddr) - fmt.Printf("[Connections] Min: %d, Max: %d, Avg: %d, Total: %d\n", conns[0], conns[num-1], avgConn, totalConn) - } -} - -type Client struct { - id uint64 - P2PClient -} - -func NewClient(i int, seeds []string) (*Client, error) { - var minOutbound = 10 - var maxConnections = 20 - - var clientIdBase = 100000 - var clientPortBase = 50000 - - client := new(Client) - client.id = uint64(clientIdBase + i) - var err error - client.P2PClient, err = GetP2PClient( - 987654321, - 1024*1024*8, - client.id, - seeds, - uint16(clientPortBase+i), - minOutbound, - maxConnections, - ) - client.P2PClient.PeerManager().SetMessageHandler(client.newHandler) - - return client, err -} - -func (c *Client) newHandler() net.MessageHandler { - return newTestHandler(c.id) -} - -type TestHandler struct { - id uint64 -} - -func newTestHandler(id uint64) net.MessageHandler { - return &TestHandler{id: id} -} - -// A handshake message received -func (c *TestHandler) OnHandshake(v *msg.Version) error { - return nil -} - -// Create a message instance by the given cmd parameter -func (c *TestHandler) MakeMessage(cmd string) (p2p.Message, error) { - var message p2p.Message - switch cmd { - case p2p.CmdPing: - message = new(msg.Ping) - case p2p.CmdPong: - message = new(msg.Pong) - } - return message, nil -} - -// VerAck message received from a connected peer -// which means the connected peer is established -func (c *TestHandler) OnPeerEstablish(peer *net.Peer) { - go c.pingWithPeer(peer) -} - -// Handle messages received from the connected peer -func (c *TestHandler) HandleMessage(peer *net.Peer, message p2p.Message) error { - switch message.(type) { - case *msg.Ping: - peer.Send(msg.NewPong(uint32(c.id))) - case *msg.Pong: - } - return nil -} - -func (c *TestHandler) pingWithPeer(peer *net.Peer) { - ticker := time.NewTicker(time.Second * 5) - for range ticker.C { - peer.Send(msg.NewPing(uint32(c.id))) - } -} diff --git a/sdk/p2pclientimpl.go b/sdk/p2pclientimpl.go deleted file mode 100644 index 7d69a9e..0000000 --- a/sdk/p2pclientimpl.go +++ /dev/null @@ -1,43 +0,0 @@ -package sdk - -import ( - "errors" - "github.com/elastos/Elastos.ELA.SPV/net" -) - -type P2PClientImpl struct { - peerManager *net.PeerManager -} - -func NewP2PClientImpl(magic, maxMsgSize uint32, clientId uint64, seeds []string, port uint16, minOutbound, maxConnections int) (*P2PClientImpl, error) { - // Initialize local peer - local := new(net.Peer) - local.SetID(clientId) - local.SetVersion(ProtocolVersion) - local.SetPort(port) - - if magic == 0 { - return nil, errors.New("Magic number has not been set ") - } - - if len(seeds) == 0 { - return nil, errors.New("Seeds list is empty ") - } - - // Create client instance - client := new(P2PClientImpl) - - // Initialize peer manager - client.peerManager = net.NewPeerManager(magic, maxMsgSize, seeds, minOutbound, maxConnections, local) - - return client, nil -} - -func (client *P2PClientImpl) Start() { - // Start - client.peerManager.Start() -} - -func (client *P2PClientImpl) PeerManager() *net.PeerManager { - return client.peerManager -} diff --git a/sdk/protocal.go b/sdk/protocal.go index b7d3465..fa24e89 100644 --- a/sdk/protocal.go +++ b/sdk/protocal.go @@ -12,7 +12,6 @@ import ( const ( ProtocolVersion = p2p.EIP001Version // The protocol version implemented SPV protocol - MaxMsgSize = 1024 * 1024 * 8 // The max size of a message in P2P network OpenService = 1 << 2 ) diff --git a/sdk/spvclient.go b/sdk/spvclient.go deleted file mode 100644 index 500cabc..0000000 --- a/sdk/spvclient.go +++ /dev/null @@ -1,68 +0,0 @@ -package sdk - -import ( - "github.com/elastos/Elastos.ELA.SPV/net" - - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" -) - -/* -SPV client will help you create and receive SPV messages, -and you will implement your own message handler to extend the SDK. -As an example, a SPV wallet implementation is in the spvwallet folder, -you can see how to extend the SDK and create your own apps. -*/ -type SPVClient interface { - // Set the message handler to extend the client - SetMessageHandler(func() SPVMessageHandler) - - // Start the client - Start() - - // Get peer manager, which is the main program of the peer to peer network - PeerManager() *net.PeerManager -} - -// The message handler to extend the SDK -type SPVMessageHandler interface { - // When a peer is connected and established - // this method will callback to pass the connected peer - OnPeerEstablish(*net.Peer) - - // After send a blocks request message, this inventory message - // will return with a bunch of block hashes, then you can use them - // to request all the blocks by send data requests. - OnInventory(*net.Peer, *msg.Inventory) error - - // After sent a data request with invType BLOCK, a merkleblock message will return through this method. - // To make this work, you must register a filterload message to the connected peer first, - // then this client will be known as a SPV client. To create a bloom filter and get the - // filterload message, you will use the method in SDK bloom sdk.NewBloomFilter() - // merkleblock includes a block header, transaction hashes in merkle proof format. - // Which transaction hashes will be in the merkleblock is depends on the addresses and outpoints - // you've added into the bloom filter before you send a filterload message with this bloom filter. - // You will use these transaction hashes to request transactions by sending data request message - // with invType TRANSACTION - OnMerkleBlock(*net.Peer, *msg.MerkleBlock) error - - // After sent a data request with invType TRANSACTION, a txn message will return through this method. - // these transactions are matched to the bloom filter you have sent with the filterload message. - OnTx(*net.Peer, *msg.Tx) error - - // If the BLOCK or TRANSACTION requested by the data request message can not be found, - // notfound message with requested data hash will return through this method. - OnNotFound(*net.Peer, *msg.NotFound) error - - // If the submitted transaction was rejected, this message will return. - OnReject(*net.Peer, *msg.Reject) error -} - -/* -Get the SPV client by set the network magic, passing the clientId and seeds arguments. -netType are TypeMainNet and TypeTestNet two options, clientId is the unique id to identify -this client in the peer to peer network. seeds is a list of other peers IP:[Port] addresses, -port is not necessary for it will be overwrite to SPVServerPort according to the SPV protocol -*/ -func GetSPVClient(magic uint32, clientId uint64, seeds []string, minOutbound, maxConnections int) (SPVClient, error) { - return NewSPVClientImpl(magic, clientId, seeds, minOutbound, maxConnections) -} diff --git a/sdk/spvclientimpl.go b/sdk/spvclientimpl.go deleted file mode 100644 index 29b7348..0000000 --- a/sdk/spvclientimpl.go +++ /dev/null @@ -1,149 +0,0 @@ -package sdk - -import ( - "errors" - "time" - - "github.com/elastos/Elastos.ELA.SPV/net" - - "github.com/elastos/Elastos.ELA.Utility/p2p" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA/core" -) - -type SPVClientImpl struct { - p2pClient P2PClient - spvMessageHandler func() SPVMessageHandler -} - -func NewSPVClientImpl(magic uint32, clientId uint64, seeds []string, minOutbound, maxConnections int) (*SPVClientImpl, error) { - // Initialize P2P client - p2pClient, err := GetP2PClient(magic, MaxMsgSize, clientId, seeds, 0, minOutbound, maxConnections) - if err != nil { - return nil, err - } - - client := &SPVClientImpl{p2pClient: p2pClient} - p2pClient.PeerManager().SetMessageHandler(client.newSpvHandler) - - return client, nil -} - -func (client *SPVClientImpl) SetMessageHandler(spvMessageHandler func() SPVMessageHandler) { - client.spvMessageHandler = spvMessageHandler -} - -func (client *SPVClientImpl) Start() { - client.p2pClient.Start() -} - -func (client *SPVClientImpl) PeerManager() *net.PeerManager { - return client.p2pClient.PeerManager() -} - -func (client *SPVClientImpl) newSpvHandler() net.MessageHandler { - return &spvHandler{ - peerManager: client.PeerManager(), - spvMessageHandler: client.spvMessageHandler(), - } -} - -type spvHandler struct { - peerManager *net.PeerManager - spvMessageHandler SPVMessageHandler -} - -// Filter peer handshake according to the SPV protocol -func (h *spvHandler) OnHandshake(v *msg.Version) error { - //if v.Version < ProtocolVersion { - // return fmt.Errorf("To support SPV protocol, peer version must greater than ", ProtocolVersion) - //} - - if v.Services/OpenService&1 == 0 { - return errors.New("SPV service not enabled on connected peer") - } - - return nil -} - -func (h *spvHandler) MakeMessage(cmd string) (message p2p.Message, err error) { - switch cmd { - case p2p.CmdPing: - message = new(msg.Ping) - case p2p.CmdPong: - message = new(msg.Pong) - case p2p.CmdInv: - message = new(msg.Inventory) - case p2p.CmdTx: - message = msg.NewTx(new(core.Transaction)) - case p2p.CmdMerkleBlock: - message = msg.NewMerkleBlock(new(core.Header)) - case p2p.CmdNotFound: - message = new(msg.NotFound) - case p2p.CmdReject: - message = new(msg.Reject) - default: - return nil, errors.New("Received unsupported message, CMD " + cmd) - } - return message, nil -} - -func (h *spvHandler) HandleMessage(peer *net.Peer, message p2p.Message) error { - switch message := message.(type) { - case *msg.Ping: - return h.OnPing(peer, message) - case *msg.Pong: - return h.OnPong(peer, message) - case *msg.Inventory: - return h.spvMessageHandler.OnInventory(peer, message) - case *msg.MerkleBlock: - return h.spvMessageHandler.OnMerkleBlock(peer, message) - case *msg.Tx: - return h.spvMessageHandler.OnTx(peer, message) - case *msg.NotFound: - return h.spvMessageHandler.OnNotFound(peer, message) - case *msg.Reject: - return h.spvMessageHandler.OnReject(peer, message) - default: - return errors.New("handle message unknown type") - } -} - -func (h *spvHandler) OnPeerEstablish(peer *net.Peer) { - h.spvMessageHandler.OnPeerEstablish(peer) - - // Start heartbeat - go h.heartBeat(peer) -} - -func (h *spvHandler) OnPing(peer *net.Peer, p *msg.Ping) error { - peer.SetHeight(p.Nonce) - // Return pong message to peer - peer.Send(msg.NewPong(uint32(h.peerManager.Local().Height()))) - return nil -} - -func (h *spvHandler) OnPong(peer *net.Peer, p *msg.Pong) error { - peer.SetHeight(p.Nonce) - return nil -} - -func (h *spvHandler) heartBeat(peer *net.Peer) { - ticker := time.NewTicker(time.Second * net.InfoUpdateDuration) - defer ticker.Stop() - for range ticker.C { - // Check if peer already disconnected - if peer.State() == p2p.INACTIVITY { - return - } - - // Disconnect peer if keep alive timeout - if time.Now().After(peer.LastActive().Add(time.Second * net.KeepAliveTimeout)) { - peer.Disconnect() - return - } - - // Send ping message to peer - peer.Send(msg.NewPing(uint32(h.peerManager.Local().Height()))) - } -} diff --git a/sdk/spvpeer.go b/sdk/spvpeer.go new file mode 100644 index 0000000..9c1de51 --- /dev/null +++ b/sdk/spvpeer.go @@ -0,0 +1,285 @@ +package sdk + +import ( + "sync" + "time" + + "github.com/elastos/Elastos.ELA.SPV/log" + "github.com/elastos/Elastos.ELA.SPV/net" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/core" +) + +const ( + // stallTickInterval is the interval of time between each check for + // stalled peers. + stallTickInterval = 15 * time.Second + + // stallResponseTimeout is the base maximum amount of time messages that + // expect a response will wait before disconnecting the peer for + // stalling. The deadlines are adjusted for callback running times and + // only checked on each stall tick interval. + stallResponseTimeout = 30 * time.Second +) + +type downloadTx struct { + mutex sync.Mutex + queue map[common.Uint256]struct{} +} + +func newDownloadTx() *downloadTx { + return &downloadTx{queue: make(map[common.Uint256]struct{})} +} + +func (d *downloadTx) queueTx(txId common.Uint256) { + d.mutex.Lock() + defer d.mutex.Unlock() + d.queue[txId] = struct{}{} +} + +func (d *downloadTx) dequeueTx(txId common.Uint256) bool { + d.mutex.Lock() + defer d.mutex.Unlock() + _, ok := d.queue[txId] + if !ok { + return false + } + delete(d.queue, txId) + return true +} + +type downloadBlock struct { + mutex sync.Mutex + *msg.MerkleBlock + txQueue map[common.Uint256]struct{} + txs []*core.Transaction +} + +func newDownloadBlock() *downloadBlock { + return &downloadBlock{txQueue: make(map[common.Uint256]struct{})} +} + +func (d *downloadBlock) enqueueTx(txId common.Uint256) { + d.mutex.Lock() + defer d.mutex.Unlock() + d.txQueue[txId] = struct{}{} +} + +func (d *downloadBlock) dequeueTx(txId common.Uint256) bool { + d.mutex.Lock() + defer d.mutex.Unlock() + _, ok := d.txQueue[txId] + if !ok { + return false + } + delete(d.txQueue, txId) + return true +} + +func (d *downloadBlock) finished() bool { + d.mutex.Lock() + defer d.mutex.Unlock() + return len(d.txQueue) == 0 +} + +type SPVPeerConfig struct { + // OnPing is invoked when peer receives a ping message. + OnPing func(*SPVPeer, *msg.Ping) + + // OnPong is invoked when peer receives a pong message. + OnPong func(*SPVPeer, *msg.Pong) + + // After send a blocks request message, this inventory message + // will return with a bunch of block hashes, then you can use them + // to request all the blocks by send data requests. + OnInventory func(*SPVPeer, *msg.Inventory) error + + // After sent a data request with invType BLOCK, a merkleblock message will return through this method. + // To make this work, you must register a filterload message to the connected peer first, + // then this client will be known as a SPV client. To create a bloom filter and get the + // filterload message, you will use the method in SDK bloom sdk.NewBloomFilter() + // merkleblock includes a block header, transaction hashes in merkle proof format. + // Which transaction hashes will be in the merkleblock is depends on the addresses and outpoints + // you've added into the bloom filter before you send a filterload message with this bloom filter. + // You will use these transaction hashes to request transactions by sending data request message + // with invType TRANSACTION + OnMerkleBlock func(*SPVPeer, *msg.MerkleBlock) error + + // After sent a data request with invType TRANSACTION, a txn message will return through this method. + // these transactions are matched to the bloom filter you have sent with the filterload message. + OnTx func(*SPVPeer, *msg.Tx) error + + // If the BLOCK or TRANSACTION requested by the data request message can not be found, + // notfound message with requested data hash will return through this method. + OnNotFound func(*SPVPeer, *msg.NotFound) error + + // If the submitted transaction was rejected, this message will return. + OnReject func(*SPVPeer, *msg.Reject) error +} + +type SPVPeer struct { + *net.Peer + + blockQueue chan common.Uint256 + downloading *downloadBlock + downloadTx *downloadTx + receivedTxs int + fPositives int + + stallControl chan p2p.Message +} + +func NewSPVPeer(peer *net.Peer, config SPVPeerConfig) *SPVPeer { + spvPeer := &SPVPeer{ + Peer: peer, + blockQueue: make(chan common.Uint256, p2p.MaxBlocksPerMsg), + downloading: newDownloadBlock(), + downloadTx: newDownloadTx(), + stallControl: make(chan p2p.Message, 1), + } + + peerConfig := net.PeerConfig{ + ProtocolVersion: p2p.EIP001Version, + MakeTx: func() *msg.Tx { return msg.NewTx(new(core.Transaction)) }, + MakeBlock: func() *msg.Block { return msg.NewBlock(new(core.Block)) }, + MakeMerkleBlock: func() *msg.MerkleBlock { return msg.NewMerkleBlock(new(core.Header)) }, + + HandleMessage: func(peer *net.Peer, message p2p.Message) { + switch m := message.(type) { + case *msg.Ping: + config.OnPing(spvPeer, m) + + case *msg.Pong: + config.OnPong(spvPeer, m) + + case *msg.Inventory: + spvPeer.stallControl <- m + config.OnInventory(spvPeer, m) + + case *msg.MerkleBlock: + spvPeer.stallControl <- m + config.OnMerkleBlock(spvPeer, m) + + case *msg.Tx: + spvPeer.stallControl <- m + config.OnTx(spvPeer, m) + + case *msg.NotFound: + spvPeer.stallControl <- m + config.OnNotFound(spvPeer, m) + + case *msg.Reject: + config.OnReject(spvPeer, m) + } + }, + } + + spvPeer.SetConfig(peerConfig) + + go spvPeer.stallHandler() + + return spvPeer +} + +func (p *SPVPeer) stallHandler() { + // stallTicker is used to periodically check pending responses that have + // exceeded the expected deadline and disconnect the peer due to stalling. + stallTicker := time.NewTicker(stallTickInterval) + defer stallTicker.Stop() + + // pendingResponses tracks the expected responses. + pendingResponses := make(map[string]struct{}) + + // lastActive tracks the last active sync message. + var lastActive time.Time + + for p.Connected() { + select { + case ctrMsg := <-p.stallControl: + // update last active time + lastActive = time.Now() + + switch message := ctrMsg.(type) { + case *msg.GetBlocks: + // Add expected response + pendingResponses[p2p.CmdInv] = struct{}{} + + case *msg.Inventory: + // Remove inventory from expected response map. + delete(pendingResponses, p2p.CmdInv) + + case *msg.GetData: + // Add expected responses + for _, iv := range message.InvList { + pendingResponses[iv.Hash.String()] = struct{}{} + } + + case *msg.MerkleBlock: + // Remove received merkleblock from expected response map. + delete(pendingResponses, message.Header.(*core.Header).Hash().String()) + + case *msg.Tx: + // Remove received transaction from expected response map. + delete(pendingResponses, message.Transaction.(*core.Transaction).Hash().String()) + + case *msg.NotFound: + // NotFound should not received from sync peer + p.Disconnect() + } + + case <-stallTicker.C: + // There are no pending responses + if len(pendingResponses) == 0 { + continue + } + + // Disconnect the peer if any of the pending responses + // don't arrive by their adjusted deadline. + if time.Now().Before(lastActive.Add(stallResponseTimeout)) { + continue + } + + log.Debugf("peer %v appears to be stalled or misbehaving, response timeout -- disconnecting", p) + p.Disconnect() + } + } + + // Drain any wait channels before going away so there is nothing left + // waiting on this goroutine. +cleanup: + for { + select { + case <-p.stallControl: + default: + break cleanup + } + } + log.Tracef("Peer stall handler done for %v", p) +} + +func (p *SPVPeer) StallMessage(message p2p.Message) { + p.stallControl <- message +} + +func (p *SPVPeer) QueueMessage(message p2p.Message, doneChan chan struct{}) { + switch message.(type) { + case *msg.GetBlocks, *msg.GetData: + p.stallControl <- message + } + p.Peer.QueueMessage(message, doneChan) +} + +func (p *SPVPeer) ResetDownloading() { + p.downloading = newDownloadBlock() +} + +func (p *SPVPeer) GetFalsePositiveRate() float32 { + return float32(p.fPositives) / float32(p.receivedTxs) +} + +func (p *SPVPeer) ResetFalsePositives() { + p.fPositives, p.receivedTxs = 0, 0 +} diff --git a/sdk/spvservice.go b/sdk/spvservice.go index 9e3ea7d..1fc6c7c 100644 --- a/sdk/spvservice.go +++ b/sdk/spvservice.go @@ -3,6 +3,7 @@ package sdk import ( "github.com/elastos/Elastos.ELA.SPV/store" + "github.com/elastos/Elastos.ELA.SPV/net" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" ela "github.com/elastos/Elastos.ELA/core" @@ -11,7 +12,7 @@ import ( /* SPV service is a high level implementation with all SPV logic implemented. SPV service is extend from SPV client and implement Blockchain and block synchronize on it. -With SPV service, you just need to implement your own HeaderStore and SPVHandler, and let other stuff go. +With SPV service, you just need to implement your own HeaderStore and SPVServiceConfig, and let other stuff go. */ type SPVService interface { // Start SPV service @@ -26,15 +27,24 @@ type SPVService interface { // ReloadFilters is a trigger to make SPV service refresh the current // transaction filer(in our implementation the bloom filter) in SPV service. - // This will call onto the GetAddresses() and GetOutpoints() method in SPVHandler. + // This will call onto the GetAddresses() and GetOutpoints() method in SPVServiceConfig. ReloadFilter() // SendTransaction broadcast a transaction message to the peer to peer network. SendTransaction(ela.Transaction) (*common.Uint256, error) } -type SPVHandler interface { - // GetData returns two arguments. +type SPVServiceConfig struct { + // The server peer access into blockchain peer to peer network + Server *net.ServerPeer + + // Foundation address of the current access blockhain network + Foundation string + + // The database to store all block headers + HeaderStore store.HeaderStore + + // GetFilterData() returns two arguments. // First arguments are all addresses stored in your data store. // Second arguments are all balance references to those addresses stored in your data store, // including UTXO(Unspent Transaction Output)s and STXO(Spent Transaction Output)s. @@ -42,7 +52,7 @@ type SPVHandler interface { // reference of an transaction output. If an address ever received an transaction output, // there will be the outpoint reference to it. Any time you want to spend the balance of an // address, you must provide the reference of the balance which is an outpoint in the transaction input. - GetData() ([]*common.Uint168, []*ela.OutPoint) + GetFilterData func() ([]*common.Uint168, []*ela.OutPoint) // When interested transactions received, this method will call back them. // The height is the block height where this transaction has been packed. @@ -50,18 +60,18 @@ type SPVHandler interface { // are not interested go through this method. If a transaction is not a match // return false as a false positive mark. If anything goes wrong, return error. // Notice: this method will be callback when commit block - CommitTx(tx *ela.Transaction, height uint32) (bool, error) + CommitTx func(tx *ela.Transaction, height uint32) (bool, error) // This method will be callback after a block and transactions with it are // successfully committed into database. - OnBlockCommitted(*msg.MerkleBlock, []*ela.Transaction) + OnBlockCommitted func(*msg.MerkleBlock, []*ela.Transaction) // When the blockchain meet a reorganization, data should be rollback to the fork point. // The Rollback method will callback the current rollback height, for example OnChainRollback(100) // means data on height 100 has been deleted, current chain height will be 99. You should rollback // stored data including UTXOs STXOs Txs etc. according to the given height. // If anything goes wrong, return an error. - OnRollback(height uint32) error + OnRollback func(height uint32) error } /* @@ -70,6 +80,6 @@ there are two implementations you need to do, DataStore and GetBloomFilter() met DataStore is an interface including all methods you need to implement placed in db/datastore.go. Also an sample APP spvwallet is contain in this project placed in spvwallet folder. */ -func GetSPVService(client SPVClient, foundation string, headerStore store.HeaderStore, handler SPVHandler) (SPVService, error) { - return NewSPVServiceImpl(client, foundation, headerStore, handler) +func GetSPVService(config SPVServiceConfig) (SPVService, error) { + return NewSPVServiceImpl(config) } diff --git a/sdk/spvserviceimpl.go b/sdk/spvserviceimpl.go index ee6486d..e69c54a 100644 --- a/sdk/spvserviceimpl.go +++ b/sdk/spvserviceimpl.go @@ -1,151 +1,131 @@ package sdk import ( + "errors" "fmt" - "sync" "time" "github.com/elastos/Elastos.ELA.SPV/log" "github.com/elastos/Elastos.ELA.SPV/net" - "github.com/elastos/Elastos.ELA.SPV/store" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA/bloom" - ela "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/core" ) const ( - SendTxTimeout = 10 - FalsePositiveRate = float32(1) / float32(1000) - SyncTickInterval = 10 * time.Second - SyncResponseTimeout = 30 * time.Second + HeartbeatInterval = time.Second * 30 + SendTxTimeout = time.Second * 10 ) -type downloadTx struct { - mutex sync.RWMutex - queue map[common.Uint256]struct{} -} - -func newDownloadTx() *downloadTx { - return &downloadTx{queue: make(map[common.Uint256]struct{})} -} - -func (d *downloadTx) queueTx(txId common.Uint256) { - d.mutex.Lock() - defer d.mutex.Unlock() - d.queue[txId] = struct{}{} -} - -func (d *downloadTx) dequeueTx(txId common.Uint256) bool { - d.mutex.Lock() - defer d.mutex.Unlock() - _, ok := d.queue[txId] - if !ok { - return false - } - delete(d.queue, txId) - return true -} - -type downloadBlock struct { - mutex sync.RWMutex - *msg.MerkleBlock - txQueue map[common.Uint256]struct{} - txs []*ela.Transaction -} - -func newDownloadBlock() *downloadBlock { - return &downloadBlock{txQueue: make(map[common.Uint256]struct{})} -} - -func (d *downloadBlock) queueTx(txId common.Uint256) { - d.mutex.Lock() - defer d.mutex.Unlock() - d.txQueue[txId] = struct{}{} -} - -func (d *downloadBlock) dequeueTx(txId common.Uint256) bool { - d.mutex.Lock() - defer d.mutex.Unlock() - _, ok := d.txQueue[txId] - if !ok { - return false - } - delete(d.txQueue, txId) - return true -} - -func (d *downloadBlock) finished() bool { - d.mutex.RLock() - defer d.mutex.RUnlock() - return len(d.txQueue) == 0 -} - // The SPV service implementation type SPVServiceImpl struct { - SPVClient + *net.ServerPeer + syncManager *SyncManager chain *Blockchain pendingTx common.Uint256 txAccept chan *common.Uint256 txReject chan *msg.Reject - handler SPVHandler - syncControl chan p2p.Message + config SPVServiceConfig } // Create a instance of SPV service implementation. -func NewSPVServiceImpl(client SPVClient, foundation string, headerStore store.HeaderStore, handler SPVHandler) (*SPVServiceImpl, error) { +func NewSPVServiceImpl(config SPVServiceConfig) (*SPVServiceImpl, error) { // Initialize blockchain - chain, err := NewBlockchain(foundation, headerStore) + chain, err := NewBlockchain(config.Foundation, config.HeaderStore) if err != nil { return nil, err } // Create SPV service instance service := &SPVServiceImpl{ - SPVClient: client, - chain: chain, - handler: handler, - syncControl: make(chan p2p.Message, 1), + ServerPeer: config.Server, + chain: chain, + config: config, } - // Set SPV handler implement - service.handler = handler + // Create sync manager config + syncConfig := SyncManageConfig{ + LocalHeight: chain.Height, + GetBlocks: service.GetBlocks, + } - // Initialize local peer height - service.updateLocalHeight(service.chain.Height()) + service.syncManager = NewSyncManager(syncConfig) - // Set p2p message handler - service.SPVClient.SetMessageHandler(service.newSpvMsgHandler) + // Set manage config + service.SetConfig(net.PeerManageConfig{ + OnHandshake: service.OnHandshake, + OnPeerEstablish: service.OnPeerEstablish, + }) return service, nil } -// Update local peer height with current chain height -func (s *SPVServiceImpl) updateLocalHeight(height uint32) { - log.Info("LocalChain height:", height) - s.PeerManager().Local().SetHeight(uint64(height)) +func (s *SPVServiceImpl) OnHandshake(v *msg.Version) error { + if v.Services/OpenService&1 == 0 { + return errors.New("SPV service not enabled on connected peer") + } + + return nil } -func (s *SPVServiceImpl) newSpvMsgHandler() SPVMessageHandler { - handler := new(spvMsgHandler) - handler.service = s +func (s *SPVServiceImpl) OnPeerEstablish(peer *net.Peer) { + // Create spv peer config + config := SPVPeerConfig{ + OnPing: s.OnPing, + OnPong: s.OnPong, + OnInventory: s.OnInventory, + OnMerkleBlock: s.OnMerkleBlock, + OnTx: s.OnTx, + OnNotFound: s.OnNotFound, + OnReject: s.OnReject, + } - // Block downloading and commit - handler.blockQueue = make(chan common.Uint256, p2p.MaxBlocksPerMsg*2) - handler.downloading = newDownloadBlock() + s.syncManager.AddNeighborPeer(NewSPVPeer(peer, config)) - // Transaction downloading - handler.downloadTx = newDownloadTx() + // Load bloom filter + doneChan := make(chan struct{}) + peer.QueueMessage(s.BloomFilter(), doneChan) + <-doneChan - return handler + // Start heartbeat + go s.heartBeat(peer) +} + +func (s *SPVServiceImpl) heartBeat(peer *net.Peer) { + ticker := time.NewTicker(HeartbeatInterval) + defer ticker.Stop() + for range ticker.C { + // Check if peer already disconnected + if !peer.Connected() { + return + } + + // Disconnect peer if keep alive timeout + if time.Now().After(peer.LastActive().Add(HeartbeatInterval * 3)) { + peer.Disconnect() + return + } + + // Send ping message to peer + peer.QueueMessage(msg.NewPing(uint32(s.chain.Height())), nil) + } +} + +func (s *SPVServiceImpl) OnPing(peer *SPVPeer, p *msg.Ping) { + peer.SetHeight(p.Nonce) + // Return pong message to peer + peer.QueueMessage(msg.NewPong(uint32(s.chain.Height())), nil) +} + +func (s *SPVServiceImpl) OnPong(peer *SPVPeer, p *msg.Pong) { + peer.SetHeight(p.Nonce) } func (s *SPVServiceImpl) Start() { - s.SPVClient.Start() - go s.syncHandler() - go s.keepUpdate() + s.ServerPeer.Start() + s.syncManager.start() log.Info("SPV service started...") } @@ -160,13 +140,13 @@ func (s *SPVServiceImpl) ChainState() ChainState { func (s *SPVServiceImpl) ReloadFilter() { log.Debug() - s.PeerManager().Broadcast(BuildBloomFilter(s.handler.GetData()).GetFilterLoadMsg()) + s.Broadcast(BuildBloomFilter(s.config.GetFilterData()).GetFilterLoadMsg()) } -func (s *SPVServiceImpl) SendTransaction(tx ela.Transaction) (*common.Uint256, error) { +func (s *SPVServiceImpl) SendTransaction(tx core.Transaction) (*common.Uint256, error) { log.Debug() - if s.PeerManager().Peers.PeersCount() == 0 { + if s.GetNeighborCount() == 0 { return nil, fmt.Errorf("method not available, no peers connected") } @@ -182,12 +162,12 @@ func (s *SPVServiceImpl) SendTransaction(tx ela.Transaction) (*common.Uint256, e // Set transaction in pending s.pendingTx = tx.Hash() // Broadcast transaction to neighbor peers - s.PeerManager().Broadcast(msg.NewTx(&tx)) + s.Broadcast(msg.NewTx(&tx)) // Query neighbors mempool see if transaction was successfully added to mempool - s.PeerManager().Broadcast(new(msg.MemPool)) + s.Broadcast(new(msg.MemPool)) // Wait for result - timer := time.NewTimer(time.Second * SendTxTimeout) + timer := time.NewTimer(SendTxTimeout) select { case <-timer.C: finish() @@ -196,7 +176,7 @@ func (s *SPVServiceImpl) SendTransaction(tx ela.Transaction) (*common.Uint256, e timer.Stop() finish() // commit unconfirmed transaction to db - _, err := s.handler.CommitTx(&tx, 0) + _, err := s.config.CommitTx(&tx, 0) return &s.pendingTx, err case msg := <-s.txReject: timer.Stop() @@ -205,210 +185,25 @@ func (s *SPVServiceImpl) SendTransaction(tx ela.Transaction) (*common.Uint256, e } } -func (s *SPVServiceImpl) keepUpdate() { - ticker := time.NewTicker(time.Second * net.InfoUpdateDuration) - defer ticker.Stop() - for range ticker.C { - // Check if blockchain need sync - if s.needSync() { - // Start syncing progress - s.startSyncing() - } else { - // Stop syncing progress - s.stopSyncing() - } - } -} - -func (s *SPVServiceImpl) needSync() bool { - // Printout neighbor peers height - peers := s.PeerManager().ConnectedPeers() - heights := make([]uint64, 0, len(peers)) - for _, peer := range peers { - heights = append(heights, peer.Height()) - } - log.Info("Neighbors -->", heights, s.PeerManager().Local().Height()) - - bestPeer := s.PeerManager().GetBestPeer() - if bestPeer == nil { // no peers connected, return false - return false - } - return bestPeer.Height() > uint64(s.chain.Height()) -} - -func (s *SPVServiceImpl) startSyncing() { - // Return if already in syncing - if s.chain.IsSyncing() { - return - } - // Get sync peer - syncPeer := s.PeerManager().GetSyncPeer() - if syncPeer == nil { - // If sync peer is nil at this point, that meas no peer connected - log.Info("no peers connected") - return - } - // Set blockchain state to syncing - s.chain.SetChainState(SYNCING) - - // Get blocks from sync peer - s.getBlocks(syncPeer) -} - -func (s *SPVServiceImpl) stopSyncing() { - // Return if not in syncing - if !s.chain.IsSyncing() { - return - } - // Set blockchain state to waiting - s.chain.SetChainState(WAITING) - // Clear sync peer - s.PeerManager().ClearSyncPeer() - // Update bloom filter - s.ReloadFilter() -} - -func (s *SPVServiceImpl) getBlocks(peer *net.Peer) { +func (s *SPVServiceImpl) GetBlocks() *msg.GetBlocks { // Get blocks returns a inventory message which contains block hashes locator := s.chain.GetBlockLocatorHashes() - getBlocks := msg.NewGetBlocks(locator, common.EmptyHash) - - s.syncControl <- getBlocks - peer.Send(getBlocks) + return msg.NewGetBlocks(locator, common.EmptyHash) } -func (s *SPVServiceImpl) syncHandler() { - // syncTicker is used to periodically check pending responses that have - // exceeded the expected deadline and disconnect the peer due to - // stalling. - syncTicker := time.NewTicker(SyncTickInterval) - defer syncTicker.Stop() - - // pendingResponses tracks the expected responses. - pendingResponses := make(map[string]struct{}) - - // lastActive tracks the last active sync message. - var lastActive time.Time - - for { - select { - case ctrMsg := <-s.syncControl: - // update last active time - lastActive = time.Now() - - switch message := ctrMsg.(type) { - case *msg.GetBlocks: - // Add expected response - pendingResponses[p2p.CmdInv] = struct{}{} - - case *msg.Inventory: - // Remove inventory from expected response map. - delete(pendingResponses, p2p.CmdInv) - - case *msg.GetData: - // Add expected responses - for _, iv := range message.InvList { - pendingResponses[iv.Hash.String()] = struct{}{} - } - - case *msg.MerkleBlock: - // Remove received merkleblock from expected response map. - delete(pendingResponses, message.Header.(*ela.Header).Hash().String()) - - case *msg.Tx: - // Remove received transaction from expected response map. - delete(pendingResponses, message.Transaction.(*ela.Transaction).Hash().String()) - - case *msg.NotFound: - // NotFound should not received from sync peer - goto QUIT - } - - case <-syncTicker.C: - // Blockchian not in syncing mode - if !s.chain.IsSyncing() { - continue - } - - // There are no pending responses - if len(pendingResponses) == 0 { - continue - } - - // Disconnect the peer if any of the pending responses - // don't arrive by their adjusted deadline. - if time.Now().Before(lastActive.Add(SyncResponseTimeout)) { - continue - } - - log.Debugf("peer %v appears to be stalled or misbehaving,"+ - " response timeout -- disconnecting", s.PeerManager().GetSyncPeer()) - goto QUIT - } - } - -QUIT: - s.changeSyncPeer() - go s.syncHandler() +func (s *SPVServiceImpl) BloomFilter() *msg.FilterLoad { + bloomFilter := BuildBloomFilter(s.config.GetFilterData()) + return bloomFilter.GetFilterLoadMsg() } -func (s *SPVServiceImpl) changeSyncPeer() { - log.Debug("Change sync peer") - syncPeer := s.PeerManager().GetSyncPeer() - if syncPeer != nil { - // Disconnect current sync peer - s.PeerManager().PeerDisconnected(syncPeer) - - // Restart - s.stopSyncing() - s.startSyncing() - } -} - -type spvMsgHandler struct { - peer *net.Peer - service *SPVServiceImpl - blockQueue chan common.Uint256 - downloading *downloadBlock - downloadTx *downloadTx - receivedTxs int - fPositives int -} - -func (h *spvMsgHandler) isSyncPeer() bool { - return h.service.chain.IsSyncing() && h.service.PeerManager().GetSyncPeer() != nil && - h.service.PeerManager().GetSyncPeer().ID() == h.peer.ID() -} - -func (h *spvMsgHandler) syncControl(msg p2p.Message) { - if h.isSyncPeer() { - h.service.syncControl <- msg - } -} - -func (h *spvMsgHandler) updateBloomFilter() { - bloomFilter := BuildBloomFilter(h.service.handler.GetData()) - h.peer.Send(bloomFilter.GetFilterLoadMsg()) -} - -func (h *spvMsgHandler) OnPeerEstablish(peer *net.Peer) { - // Set handler's peer - h.peer = peer - // Send filterload message - h.updateBloomFilter() -} - -func (h *spvMsgHandler) OnInventory(peer *net.Peer, m *msg.Inventory) error { - // Notify sync control - h.syncControl(m) - +func (s *SPVServiceImpl) OnInventory(peer *SPVPeer, m *msg.Inventory) error { getData := msg.NewGetData() for _, inv := range m.InvList { switch inv.Type { case msg.InvTypeBlock: // Filter duplicated block - if h.service.chain.IsKnownHeader(&inv.Hash) { + if s.chain.IsKnownHeader(&inv.Hash) { continue } @@ -417,17 +212,17 @@ func (h *spvMsgHandler) OnInventory(peer *net.Peer, m *msg.Inventory) error { // need separate timeout handling. inv.Type = msg.InvTypeFilteredBlock getData.AddInvVect(inv) - if h.isSyncPeer() { - h.blockQueue <- inv.Hash + if s.syncManager.IsSyncPeer(peer) { + peer.blockQueue <- inv.Hash } case msg.InvTypeTx: - if h.service.txAccept != nil && h.service.pendingTx.IsEqual(inv.Hash) { - h.service.txAccept <- nil + if s.txAccept != nil && s.pendingTx.IsEqual(inv.Hash) { + s.txAccept <- nil continue } getData.AddInvVect(inv) - h.downloadTx.queueTx(inv.Hash) + peer.downloadTx.queueTx(inv.Hash) default: continue @@ -435,25 +230,21 @@ func (h *spvMsgHandler) OnInventory(peer *net.Peer, m *msg.Inventory) error { } if len(getData.InvList) > 0 { - // Notify sync control - h.syncControl(getData) - - peer.Send(getData) + doneChan := make(chan struct{}) + peer.QueueMessage(getData, doneChan) + <-doneChan } return nil } -func (h *spvMsgHandler) OnMerkleBlock(peer *net.Peer, block *msg.MerkleBlock) error { - // Notify sync control - h.syncControl(block) - - blockHash := block.Header.(*ela.Header).Hash() +func (s *SPVServiceImpl) OnMerkleBlock(peer *SPVPeer, block *msg.MerkleBlock) error { + blockHash := block.Header.(*core.Header).Hash() // Merkleblock from sync peer - if h.isSyncPeer() { - queueHash := <-h.blockQueue + if s.syncManager.IsSyncPeer(peer) { + queueHash := <-peer.blockQueue if !blockHash.IsEqual(queueHash) { - h.changeSyncPeer() + peer.Disconnect() return fmt.Errorf("peer %d is sending us blocks out of order", peer.ID()) } } @@ -464,11 +255,11 @@ func (h *spvMsgHandler) OnMerkleBlock(peer *net.Peer, block *msg.MerkleBlock) er } // Save block as download block - h.downloading.MerkleBlock = block + peer.downloading.MerkleBlock = block // No transactions to download, just finish it if len(txIds) == 0 { - h.finishDownload() + s.finishDownloading(peer) return nil } @@ -476,137 +267,116 @@ func (h *spvMsgHandler) OnMerkleBlock(peer *net.Peer, block *msg.MerkleBlock) er getData := msg.NewGetData() for _, txId := range txIds { getData.AddInvVect(msg.NewInvVect(msg.InvTypeTx, txId)) - h.downloading.queueTx(*txId) + peer.downloading.enqueueTx(*txId) } - // Notify sync control - h.syncControl(getData) + // Stall message + peer.StallMessage(getData) return nil } -func (h *spvMsgHandler) OnTx(peer *net.Peer, msg *msg.Tx) error { - // Notify sync control - h.syncControl(msg) - - tx := msg.Transaction.(*ela.Transaction) - if h.downloadTx.dequeueTx(tx.Hash()) { +func (s *SPVServiceImpl) OnTx(peer *SPVPeer, msg *msg.Tx) error { + tx := msg.Transaction.(*core.Transaction) + if peer.downloadTx.dequeueTx(tx.Hash()) { // commit unconfirmed transaction - _, err := h.service.handler.CommitTx(tx, 0) + _, err := s.config.CommitTx(tx, 0) if err == nil { - h.updateBloomFilter() + // Update bloom filter + doneChan := make(chan struct{}) + peer.QueueMessage(s.BloomFilter(), doneChan) + <-doneChan } return err } - if !h.downloading.dequeueTx(tx.Hash()) { - h.downloading = newDownloadBlock() + if !peer.downloading.dequeueTx(tx.Hash()) { + peer.downloading = newDownloadBlock() return fmt.Errorf("Transaction not found in download queue %s", tx.Hash().String()) } // Add tx to download - h.downloading.txs = append(h.downloading.txs, tx) + peer.downloading.txs = append(peer.downloading.txs, tx) // All transactions of the download block have been received, commit the download block - if h.downloading.finished() { - h.finishDownload() + if peer.downloading.finished() { + // Finish current downloading block + s.finishDownloading(peer) + } return nil } -func (h *spvMsgHandler) OnNotFound(peer *net.Peer, notFound *msg.NotFound) error { - // Notify sync control - h.syncControl(notFound) - +func (s *SPVServiceImpl) OnNotFound(peer *SPVPeer, notFound *msg.NotFound) error { for _, iv := range notFound.InvList { log.Warnf("Data not found type %s, hash %s", iv.Type.String(), iv.Hash.String()) switch iv.Type { case msg.InvTypeTx: - if h.downloadTx.dequeueTx(iv.Hash) { + if peer.downloadTx.dequeueTx(iv.Hash) { } - if h.downloading.dequeueTx(iv.Hash) { - h.downloading = newDownloadBlock() + if peer.downloading.dequeueTx(iv.Hash) { + peer.ResetDownloading() return nil } case msg.InvTypeBlock: - h.downloading = newDownloadBlock() + peer.ResetDownloading() } } return nil } -func (h *spvMsgHandler) OnReject(peer *net.Peer, msg *msg.Reject) error { - if h.service.pendingTx.IsEqual(msg.Hash); h.service.txReject != nil { - h.service.txReject <- msg +func (s *SPVServiceImpl) OnReject(peer *SPVPeer, msg *msg.Reject) error { + if s.pendingTx.IsEqual(msg.Hash); s.txReject != nil { + s.txReject <- msg return nil } return fmt.Errorf("Received reject message from peer %d: Code: %s, Hash %s, Reason: %s", peer.ID(), msg.Code.String(), msg.Hash.String(), msg.Reason) } -func (h *spvMsgHandler) changeSyncPeer() { - // Reset downloading block - h.downloading = newDownloadBlock() - // Reset downloading transaction - h.downloadTx = newDownloadTx() - - // Clear download block queue - for len(h.blockQueue) > 0 { - <-h.blockQueue - } +func (s *SPVServiceImpl) finishDownloading(peer *SPVPeer) { + // Commit downloaded block + s.commitBlock(peer) - // Change sync peer - h.service.changeSyncPeer() -} + peer.ResetDownloading() -func (h *spvMsgHandler) finishDownload() { - // Commit downloaded block - h.commitBlock(h.downloading) - h.downloading = newDownloadBlock() - // Request next block list when in syncing - if h.isSyncPeer() && len(h.blockQueue) == 0 { - // Get more blocks - h.service.getBlocks(h.peer) - } + s.syncManager.ContinueSync() } -func (h *spvMsgHandler) commitBlock(block *downloadBlock) { - header := block.Header.(*ela.Header) - newTip, reorgFrom, err := h.service.chain.CommitHeader(*header) +func (s *SPVServiceImpl) commitBlock(peer *SPVPeer) { + block := peer.downloading + header := block.Header.(*core.Header) + newTip, reorgFrom, err := s.chain.CommitHeader(*header) if err != nil { log.Errorf("Commit header failed %s", err.Error()) - // If a syncing peer send us bad block, disconnect it. - if h.isSyncPeer() { - h.changeSyncPeer() - } return } if !newTip { return } - newHeight := h.service.chain.Height() + newHeight := s.chain.Height() if reorgFrom > 0 { for i := reorgFrom; i > newHeight; i-- { - if err = h.service.handler.OnRollback(i); err != nil { + if err = s.config.OnRollback(i); err != nil { log.Errorf("Rollback transaction at height %d failed %s", i, err.Error()) return } } - if !h.service.chain.IsSyncing() { - h.service.startSyncing() + if !s.chain.IsSyncing() { + s.syncManager.StartSyncing() return } } for _, tx := range block.txs { // Increase received transaction count - h.receivedTxs++ + peer.receivedTxs++ - falsePositive, err := h.service.handler.CommitTx(tx, header.Height) + falsePositive, err := s.config.CommitTx(tx, header.Height) if err != nil { log.Errorf("Commit transaction %s failed %s", tx.Hash().String(), err.Error()) return @@ -614,17 +384,20 @@ func (h *spvMsgHandler) commitBlock(block *downloadBlock) { // Increase false positive count if falsePositive { - h.fPositives++ + peer.fPositives++ } } // Refresh bloom filter if false positives meet target rate - if float32(h.fPositives)/float32(h.receivedTxs) > FalsePositiveRate { - h.updateBloomFilter() - h.receivedTxs = 0 - h.fPositives = 0 + if peer.GetFalsePositiveRate() > FalsePositiveRate { + // Reset false positives + peer.ResetFalsePositives() + + // Update bloom filter + doneChan := make(chan struct{}) + peer.QueueMessage(s.BloomFilter(), doneChan) + <-doneChan } - h.service.updateLocalHeight(newHeight) - h.service.handler.OnBlockCommitted(block.MerkleBlock, block.txs) + s.config.OnBlockCommitted(block.MerkleBlock, block.txs) } diff --git a/sdk/syncmanager.go b/sdk/syncmanager.go new file mode 100644 index 0000000..c05cb0a --- /dev/null +++ b/sdk/syncmanager.go @@ -0,0 +1,186 @@ +package sdk + +import ( + "sync" + "time" + + "github.com/elastos/Elastos.ELA.SPV/log" + + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" +) + +const ( + SyncTickInterval = time.Second * 5 + FalsePositiveRate = float32(1) / float32(1000) +) + +type neighbors struct { + sync.Mutex + list map[uint64]*SPVPeer +} + +func (ns *neighbors) init() { + ns.list = make(map[uint64]*SPVPeer) +} + +func (ns *neighbors) addNeighbor(peer *SPVPeer) { + // Add peer to list + ns.Lock() + ns.list[peer.ID()] = peer + ns.Unlock() +} + +func (ns *neighbors) delNeighbor(id uint64) { + ns.Lock() + delete(ns.list, id) + ns.Unlock() +} + +func (ns *neighbors) getNeighborPeers() []*SPVPeer { + ns.Lock() + defer ns.Unlock() + + peers := make([]*SPVPeer, 0, len(ns.list)) + for _, peer := range ns.list { + if !peer.Connected() { + continue + } + + peers = append(peers, peer) + } + + return peers +} + +func (ns *neighbors) getBestPeer() *SPVPeer { + ns.Lock() + defer ns.Unlock() + var best *SPVPeer + for _, peer := range ns.list { + // Skip disconnected peer + if !peer.Connected() { + continue + } + + // Init best peer + if best == nil { + best = peer + continue + } + + if peer.Height() > best.Height() { + best = peer + } + } + + return best +} + +type SyncManageConfig struct { + LocalHeight func() uint32 + GetBlocks func() *msg.GetBlocks +} + +type SyncManager struct { + config SyncManageConfig + syncPeer *SPVPeer + neighbors +} + +func NewSyncManager(config SyncManageConfig) *SyncManager { + return &SyncManager{config: config} +} + +func (s *SyncManager) start() { + // Initial neighbor list + s.neighbors.init() + + // Start sync handler + go s.syncHandler() +} + +func (s *SyncManager) syncHandler() { + // Check if need sync by SyncTickInterval + ticker := time.NewTicker(SyncTickInterval) + defer ticker.Stop() + + for range ticker.C { + // Try to start a syncing progress + s.StartSyncing() + } +} + +func (s *SyncManager) needSync() (*SPVPeer, bool) { + // Printout neighbor peers height + peers := s.getNeighborPeers() + heights := make([]uint64, 0, len(peers)) + for _, peer := range peers { + heights = append(heights, peer.Height()) + } + log.Info("Neighbors -->", heights, s.config.LocalHeight()) + + bestPeer := s.getBestPeer() + if bestPeer == nil { // no peers connected, return false + log.Info("no peers connected") + return nil, false + } + return bestPeer, bestPeer.Height() > uint64(s.config.LocalHeight()) +} + +func (s *SyncManager) getBlocks() { + doneChan := make(chan struct{}) + s.syncPeer.QueueMessage(s.config.GetBlocks(), doneChan) + <-doneChan +} + +func (s *SyncManager) AddNeighborPeer(peer *SPVPeer) { + // Wait for peer quit + go func() { + select { + case <-peer.QuitChan(): + if s.syncPeer != nil && s.syncPeer.ID() == peer.ID() { + s.syncPeer = nil + } + s.delNeighbor(peer.ID()) + } + }() + + // Set handler's peer + s.addNeighbor(peer) +} + +func (s *SyncManager) StartSyncing() { + // Check if blockchain need sync + if bestPeer, needSync := s.needSync(); needSync { + // Return if already in syncing + if s.syncPeer != nil { + return + } + + // Set sync peer + s.syncPeer = bestPeer + + // Send getblocks to sync peer + s.getBlocks() + + } else { + // Return if not in syncing + if s.syncPeer == nil { + return + } + + // Clear sync peer + s.syncPeer = nil + + } +} + +func (s *SyncManager) ContinueSync() { + if s.syncPeer != nil && len(s.syncPeer.blockQueue) == 0 { + s.getBlocks() + } +} + +func (s *SyncManager) IsSyncPeer(peer *SPVPeer) bool { + return s.syncPeer != nil && s.syncPeer.ID() == peer.ID() +} diff --git a/spvwallet/spvwallet.go b/spvwallet/spvwallet.go index b7fe5ac..2d004fb 100644 --- a/spvwallet/spvwallet.go +++ b/spvwallet/spvwallet.go @@ -10,7 +10,9 @@ import ( "github.com/elastos/Elastos.ELA.SPV/spvwallet/db" "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" + "github.com/elastos/Elastos.ELA.SPV/net" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA/core" ) @@ -40,14 +42,27 @@ func Init(clientId uint64, seeds []string) (*SPVWallet, error) { // Initialize txs cache wallet.txIds = NewTxIdCache(MaxTxIdCached) - // Initialize P2P network client - client, err := sdk.GetSPVClient(config.Values().Magic, clientId, seeds, MaxConnections, MaxConnections) - if err != nil { - return nil, err + // Create server peer config + serverPeerConfig := net.ServerPeerConfig{ + Magic: config.Values().Magic, + Version: p2p.EIP001Version, + PeerId: clientId, + Port: 0, + Seeds: seeds, + MinOutbound: MaxConnections, + MaxConnections: MaxConnections, } // Initialize spv service - wallet.SPVService, err = sdk.GetSPVService(client, config.Values().Foundation, wallet.headerStore, wallet) + wallet.SPVService, err = sdk.GetSPVService(sdk.SPVServiceConfig{ + Server: net.NewServerPeer(serverPeerConfig), + Foundation: config.Values().Foundation, + HeaderStore: wallet.headerStore, + GetFilterData: wallet.GetFilterData, + CommitTx: wallet.CommitTx, + OnBlockCommitted: wallet.OnBlockCommitted, + OnRollback: wallet.OnRollback, + }) if err != nil { return nil, err } @@ -86,7 +101,7 @@ func (wallet *SPVWallet) Stop() { wallet.rpcServer.Close() } -func (wallet *SPVWallet) GetData() ([]*common.Uint168, []*core.OutPoint) { +func (wallet *SPVWallet) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { utxos, _ := wallet.dataStore.UTXOs().GetAll() stxos, _ := wallet.dataStore.STXOs().GetAll() From 29928267dd734c2d31cf65588fd9fc01a688b605 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 20 Aug 2018 15:29:11 +0800 Subject: [PATCH 02/73] fix peer disconnect by hearbeat timeout issue --- net/peer.go | 134 +++++++++++++++++++++++++++++++++++------- net/serverpeer.go | 17 +++--- sdk/spvpeer.go | 41 +++++++------ sdk/spvserviceimpl.go | 40 ++----------- 4 files changed, 152 insertions(+), 80 deletions(-) diff --git a/net/peer.go b/net/peer.go index cb0c98b..5b8e5f6 100644 --- a/net/peer.go +++ b/net/peer.go @@ -21,7 +21,11 @@ const ( outputBufferSize = 50 // idleTimeout is the duration of inactivity before we time out a peer. - idleTimeout = 5 * time.Minute + idleTimeout = 2 * time.Minute + + // pingInterval is the interval of time to wait in between sending ping + // messages. + pingInterval = 30 * time.Second ) // outMsg is used to house a message to be sent along with a channel to signal @@ -33,11 +37,14 @@ type outMsg struct { } type PeerConfig struct { - ProtocolVersion uint32 // The P2P network protocol version - MakeTx func() *msg.Tx - MakeBlock func() *msg.Block - MakeMerkleBlock func() *msg.MerkleBlock - HandleMessage func(peer *Peer, msg p2p.Message) + PingNonce func() uint32 + PongNonce func() uint32 + OnVerAck func(peer *Peer) + OnGetAddr func(peer *Peer) + OnAddr func(peer *Peer, addr *msg.Addr) + OnPing func(peer *Peer, ping *msg.Ping) + OnPong func(peer *Peer, pong *msg.Pong) + HandleMessage func(peer *Peer, msg p2p.Message) } type Peer struct { @@ -54,7 +61,7 @@ type Peer struct { conn net.Conn rw rw.MessageRW - handleMessage func(peer *Peer, msg p2p.Message) + config PeerConfig outputQueue chan outMsg sendQueue chan outMsg sendDoneQueue chan struct{} @@ -235,7 +242,41 @@ out: log.Debugf("-----> inHandler [%s] from [0x%x]", rmsg.CMD(), p.id) // Handle each message. - p.handleMessage(p, rmsg) + switch m := rmsg.(type) { + case *msg.VerAck: + if p.config.OnVerAck != nil { + p.config.OnVerAck(p) + } + + case *msg.GetAddr: + if p.config.OnGetAddr != nil { + p.config.OnGetAddr(p) + } + + case *msg.Addr: + if p.config.OnAddr != nil { + p.config.OnAddr(p, m) + } + + case *msg.Ping: + if p.config.PongNonce != nil { + p.QueueMessage(msg.NewPong(p.config.PongNonce()), nil) + } + + if p.config.OnPing != nil { + p.config.OnPing(p, m) + } + + case *msg.Pong: + if p.config.OnPong != nil { + p.config.OnPong(p, m) + } + + default: + if p.config.HandleMessage != nil { + p.config.HandleMessage(p, rmsg) + } + } // A message was received so reset the idle timer. idleTimer.Reset(idleTimeout) @@ -370,6 +411,23 @@ cleanup: log.Tracef("Peer output handler done for %s", p) } +// pingHandler periodically pings the peer. It must be run as a goroutine. +func (p *Peer) pingHandler() { + pingTicker := time.NewTicker(pingInterval) + defer pingTicker.Stop() + +out: + for { + select { + case <-pingTicker.C: + p.QueueMessage(msg.NewPing(p.config.PingNonce()), nil) + + case <-p.quit: + break out + } + } +} + func (p *Peer) readMessage() (p2p.Message, error) { return p.rw.ReadMessage(p.conn) } @@ -399,6 +457,7 @@ func (p *Peer) start() { go p.inHandler() go p.queueHandler() go p.outHandler() + go p.pingHandler() } func (p *Peer) NewVersionMsg() *msg.Version { @@ -413,21 +472,56 @@ func (p *Peer) NewVersionMsg() *msg.Version { return version } -func (p *Peer) SetConfig(config PeerConfig) { - rwConfig := rw.MessageConfig{ - ProtocolVersion: config.ProtocolVersion, - MakeTx: config.MakeTx, - MakeBlock: config.MakeBlock, - MakeMerkleBlock: config.MakeMerkleBlock, +func (p *Peer) SetPeerConfig(config PeerConfig) { + // Set PingNonce method + if config.PingNonce != nil { + p.config.PingNonce = config.PingNonce + } + + // Set OnVerAck method + if config.OnVerAck != nil { + p.config.OnVerAck = config.OnVerAck + } + + // Set OnGetAddr method + if config.OnGetAddr != nil { + p.config.OnGetAddr = config.OnGetAddr + } + + // Set OnGetAddr method + if config.OnAddr != nil { + p.config.OnAddr = config.OnAddr } - p.rw.SetConfig(rwConfig) - // Upgrade peer message handler - previousHandler := p.handleMessage - p.handleMessage = func(peer *Peer, msg p2p.Message) { - previousHandler(peer, msg) - config.HandleMessage(peer, msg) + // Set OnPing method + if config.OnPing != nil { + p.config.OnPing = config.OnPing } + + // Set OnPong method + if config.OnPong != nil { + p.config.OnPong = config.OnPong + } + + if config.HandleMessage != nil { + if p.config.HandleMessage == nil { + // Set message handler + p.config.HandleMessage = config.HandleMessage + + } else { + // Upgrade peer message handler + previousHandler := p.config.HandleMessage + p.config.HandleMessage = func(peer *Peer, msg p2p.Message) { + previousHandler(peer, msg) + config.HandleMessage(peer, msg) + } + } + } +} + +func (p *Peer) SetMessageConfig(config rw.MessageConfig) { + // Set rw config + p.rw.SetConfig(config) } func NewPeer(magic uint32, conn net.Conn) *Peer { diff --git a/net/serverpeer.go b/net/serverpeer.go index e9b9317..f747d0e 100644 --- a/net/serverpeer.go +++ b/net/serverpeer.go @@ -178,7 +178,7 @@ func (sp *ServerPeer) handshake(peer *Peer, inbound bool, doneChan chan struct{} doneChan <- struct{}{} // Update peer's message config - peer.handleMessage = sp.baseMessageHandler() + peer.SetPeerConfig(sp.basePeerConfig()) // Start peer peer.start() @@ -237,17 +237,18 @@ func (sp *ServerPeer) keepConnections() { } } -func (sp *ServerPeer) baseMessageHandler() func(peer *Peer, message p2p.Message) { - return func(peer *Peer, message p2p.Message) { - switch m := message.(type) { - case *msg.VerAck: +func (sp *ServerPeer) basePeerConfig() PeerConfig { + return PeerConfig{ + OnVerAck: func(peer *Peer) { // Notify peer establish sp.config.OnPeerEstablish(peer) + }, - case *msg.GetAddr: + OnGetAddr: func(peer *Peer) { peer.QueueMessage(msg.NewAddr(sp.am.RandGetAddresses()), nil) + }, - case *msg.Addr: + OnAddr: func(peer *Peer, m *msg.Addr) { for _, addr := range m.AddrList { // Skip local peer if addr.ID == sp.ID() { @@ -264,7 +265,7 @@ func (sp *ServerPeer) baseMessageHandler() func(peer *Peer, message p2p.Message) // Save to address list sp.am.AddOrUpdateAddress(&addr) } - } + }, } } diff --git a/sdk/spvpeer.go b/sdk/spvpeer.go index 9c1de51..29c8e49 100644 --- a/sdk/spvpeer.go +++ b/sdk/spvpeer.go @@ -10,6 +10,7 @@ import ( "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA.Utility/p2p/rw" "github.com/elastos/Elastos.ELA/core" ) @@ -86,11 +87,8 @@ func (d *downloadBlock) finished() bool { } type SPVPeerConfig struct { - // OnPing is invoked when peer receives a ping message. - OnPing func(*SPVPeer, *msg.Ping) - - // OnPong is invoked when peer receives a pong message. - OnPong func(*SPVPeer, *msg.Pong) + // LocalHeight is invoked when peer queue a ping or pong message + LocalHeight func() uint32 // After send a blocks request message, this inventory message // will return with a bunch of block hashes, then you can use them @@ -141,34 +139,43 @@ func NewSPVPeer(peer *net.Peer, config SPVPeerConfig) *SPVPeer { stallControl: make(chan p2p.Message, 1), } - peerConfig := net.PeerConfig{ + msgConfig := rw.MessageConfig{ ProtocolVersion: p2p.EIP001Version, MakeTx: func() *msg.Tx { return msg.NewTx(new(core.Transaction)) }, MakeBlock: func() *msg.Block { return msg.NewBlock(new(core.Block)) }, MakeMerkleBlock: func() *msg.MerkleBlock { return msg.NewMerkleBlock(new(core.Header)) }, + } - HandleMessage: func(peer *net.Peer, message p2p.Message) { - switch m := message.(type) { - case *msg.Ping: - config.OnPing(spvPeer, m) + spvPeer.SetMessageConfig(msgConfig) + + peerConfig := net.PeerConfig{ + PingNonce: config.LocalHeight, - case *msg.Pong: - config.OnPong(spvPeer, m) + PongNonce: config.LocalHeight, + OnPing: func(peer *net.Peer, ping *msg.Ping) { + peer.SetHeight(ping.Nonce) + }, + + OnPong: func(peer *net.Peer, pong *msg.Pong) { + peer.SetHeight(pong.Nonce) + }, + + HandleMessage: func(peer *net.Peer, message p2p.Message) { + // Notify stall control + spvPeer.stallControl <- message + + switch m := message.(type) { case *msg.Inventory: - spvPeer.stallControl <- m config.OnInventory(spvPeer, m) case *msg.MerkleBlock: - spvPeer.stallControl <- m config.OnMerkleBlock(spvPeer, m) case *msg.Tx: - spvPeer.stallControl <- m config.OnTx(spvPeer, m) case *msg.NotFound: - spvPeer.stallControl <- m config.OnNotFound(spvPeer, m) case *msg.Reject: @@ -177,7 +184,7 @@ func NewSPVPeer(peer *net.Peer, config SPVPeerConfig) *SPVPeer { }, } - spvPeer.SetConfig(peerConfig) + spvPeer.SetPeerConfig(peerConfig) go spvPeer.stallHandler() diff --git a/sdk/spvserviceimpl.go b/sdk/spvserviceimpl.go index e69c54a..53e30d3 100644 --- a/sdk/spvserviceimpl.go +++ b/sdk/spvserviceimpl.go @@ -15,8 +15,7 @@ import ( ) const ( - HeartbeatInterval = time.Second * 30 - SendTxTimeout = time.Second * 10 + SendTxTimeout = time.Second * 10 ) // The SPV service implementation @@ -73,8 +72,7 @@ func (s *SPVServiceImpl) OnHandshake(v *msg.Version) error { func (s *SPVServiceImpl) OnPeerEstablish(peer *net.Peer) { // Create spv peer config config := SPVPeerConfig{ - OnPing: s.OnPing, - OnPong: s.OnPong, + LocalHeight: s.LocalHeight, OnInventory: s.OnInventory, OnMerkleBlock: s.OnMerkleBlock, OnTx: s.OnTx, @@ -88,39 +86,10 @@ func (s *SPVServiceImpl) OnPeerEstablish(peer *net.Peer) { doneChan := make(chan struct{}) peer.QueueMessage(s.BloomFilter(), doneChan) <-doneChan - - // Start heartbeat - go s.heartBeat(peer) -} - -func (s *SPVServiceImpl) heartBeat(peer *net.Peer) { - ticker := time.NewTicker(HeartbeatInterval) - defer ticker.Stop() - for range ticker.C { - // Check if peer already disconnected - if !peer.Connected() { - return - } - - // Disconnect peer if keep alive timeout - if time.Now().After(peer.LastActive().Add(HeartbeatInterval * 3)) { - peer.Disconnect() - return - } - - // Send ping message to peer - peer.QueueMessage(msg.NewPing(uint32(s.chain.Height())), nil) - } -} - -func (s *SPVServiceImpl) OnPing(peer *SPVPeer, p *msg.Ping) { - peer.SetHeight(p.Nonce) - // Return pong message to peer - peer.QueueMessage(msg.NewPong(uint32(s.chain.Height())), nil) } -func (s *SPVServiceImpl) OnPong(peer *SPVPeer, p *msg.Pong) { - peer.SetHeight(p.Nonce) +func (s *SPVServiceImpl) LocalHeight() uint32 { + return uint32(s.ServerPeer.Height()) } func (s *SPVServiceImpl) Start() { @@ -399,5 +368,6 @@ func (s *SPVServiceImpl) commitBlock(peer *SPVPeer) { <-doneChan } + s.ServerPeer.SetHeight(uint64(newHeight)) s.config.OnBlockCommitted(block.MerkleBlock, block.txs) } From 25411bce903c9d119e76afae35063ea303226b40 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 20 Aug 2018 15:45:08 +0800 Subject: [PATCH 03/73] add SendMessage() to SPVPeer to handle synchronized message sending --- sdk/spvpeer.go | 14 +++++++++++++- sdk/spvserviceimpl.go | 12 +++--------- sdk/syncmanager.go | 10 ++++------ 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/sdk/spvpeer.go b/sdk/spvpeer.go index 29c8e49..12f384a 100644 --- a/sdk/spvpeer.go +++ b/sdk/spvpeer.go @@ -271,7 +271,19 @@ func (p *SPVPeer) StallMessage(message p2p.Message) { p.stallControl <- message } -func (p *SPVPeer) QueueMessage(message p2p.Message, doneChan chan struct{}) { +// Add message to output queue and wait until message sent +func (p *SPVPeer) SendMessage(message p2p.Message) { + doneChan := make(chan struct{}) + p.queueMessage(message, doneChan) + <-doneChan +} + +// Add a message into output queue +func (p *SPVPeer) QueueMessage(message p2p.Message) { + p.queueMessage(message, nil) +} + +func (p *SPVPeer) queueMessage(message p2p.Message, doneChan chan struct{}) { switch message.(type) { case *msg.GetBlocks, *msg.GetData: p.stallControl <- message diff --git a/sdk/spvserviceimpl.go b/sdk/spvserviceimpl.go index 53e30d3..e2acc47 100644 --- a/sdk/spvserviceimpl.go +++ b/sdk/spvserviceimpl.go @@ -199,9 +199,7 @@ func (s *SPVServiceImpl) OnInventory(peer *SPVPeer, m *msg.Inventory) error { } if len(getData.InvList) > 0 { - doneChan := make(chan struct{}) - peer.QueueMessage(getData, doneChan) - <-doneChan + peer.QueueMessage(getData) } return nil } @@ -251,9 +249,7 @@ func (s *SPVServiceImpl) OnTx(peer *SPVPeer, msg *msg.Tx) error { _, err := s.config.CommitTx(tx, 0) if err == nil { // Update bloom filter - doneChan := make(chan struct{}) - peer.QueueMessage(s.BloomFilter(), doneChan) - <-doneChan + peer.SendMessage(s.BloomFilter()) } return err } @@ -363,9 +359,7 @@ func (s *SPVServiceImpl) commitBlock(peer *SPVPeer) { peer.ResetFalsePositives() // Update bloom filter - doneChan := make(chan struct{}) - peer.QueueMessage(s.BloomFilter(), doneChan) - <-doneChan + peer.SendMessage(s.BloomFilter()) } s.ServerPeer.SetHeight(uint64(newHeight)) diff --git a/sdk/syncmanager.go b/sdk/syncmanager.go index c05cb0a..d2f56a8 100644 --- a/sdk/syncmanager.go +++ b/sdk/syncmanager.go @@ -127,10 +127,8 @@ func (s *SyncManager) needSync() (*SPVPeer, bool) { return bestPeer, bestPeer.Height() > uint64(s.config.LocalHeight()) } -func (s *SyncManager) getBlocks() { - doneChan := make(chan struct{}) - s.syncPeer.QueueMessage(s.config.GetBlocks(), doneChan) - <-doneChan +func (s *SyncManager) GetBlocks() { + s.syncPeer.QueueMessage(s.config.GetBlocks()) } func (s *SyncManager) AddNeighborPeer(peer *SPVPeer) { @@ -161,7 +159,7 @@ func (s *SyncManager) StartSyncing() { s.syncPeer = bestPeer // Send getblocks to sync peer - s.getBlocks() + s.GetBlocks() } else { // Return if not in syncing @@ -177,7 +175,7 @@ func (s *SyncManager) StartSyncing() { func (s *SyncManager) ContinueSync() { if s.syncPeer != nil && len(s.syncPeer.blockQueue) == 0 { - s.getBlocks() + s.GetBlocks() } } From be0f8d6eb65d531ca790d27611ad17fbeb1e71e6 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 8 Sep 2018 13:30:33 +0800 Subject: [PATCH 04/73] initial blockchain package implement --- blockchain/chain.go | 277 +++++++++ blockchain/difficulty.go | 86 +++ blockchain/genesis.go | 85 +++ blockchain/log.go | 28 + database/headers.go | 27 + net/addrmanager.go | 184 ------ net/connmanager.go | 130 ----- net/knowaddress.go | 107 ---- net/neighbors.go | 72 --- net/peer.go | 548 ------------------ net/serverpeer.go | 295 ---------- sdk/spvpeer.go => peer/peer.go | 116 ++-- sdk/blockchain.go | 373 ------------ sdk/interface.go | 81 +++ sdk/spvservice.go | 400 ++++++++++--- sdk/spvserviceimpl.go | 367 ------------ sdk/syncmanager.go | 184 ------ spvwallet/{cli => client}/account/account.go | 11 +- spvwallet/{cli => client}/common.go | 17 +- spvwallet/client/database/database.go | 110 ++++ spvwallet/client/interface.go | 382 ++++++++++++ spvwallet/{ => client}/keystore.go | 0 spvwallet/{ => client}/keystore_file.go | 0 .../transaction/transaction.go | 31 +- spvwallet/{cli => client}/wallet/wallet.go | 4 +- spvwallet/database.go | 113 ---- spvwallet/db/chain.go | 52 -- spvwallet/db/datastore.go | 94 --- spvwallet/spvwallet.go | 282 --------- .../headers.go => store/headers/database.go} | 97 +--- .../{db/addrsdb.go => store/sqlite/addrs.go} | 24 +- .../sqlitedb.go => store/sqlite/database.go} | 150 ++++- spvwallet/store/sqlite/interface.go | 144 +++++ spvwallet/store/sqlite/state.go | 52 ++ .../{db/stxosdb.go => store/sqlite/stxos.go} | 31 +- .../{db/txsdb.go => store/sqlite/txs.go} | 42 +- .../{db/utxosdb.go => store/sqlite/utxos.go} | 30 +- spvwallet/{db => sutil}/addr.go | 2 +- spvwallet/{db => sutil}/stxo.go | 2 +- spvwallet/{db => sutil}/tx.go | 2 +- spvwallet/{db => sutil}/utxo.go | 2 +- spvwallet/wallet.go | 528 +++++++---------- store/headerstore.go | 23 - store/storeheader.go => util/header.go | 15 +- 44 files changed, 2156 insertions(+), 3444 deletions(-) create mode 100644 blockchain/chain.go create mode 100644 blockchain/difficulty.go create mode 100644 blockchain/genesis.go create mode 100644 blockchain/log.go create mode 100644 database/headers.go delete mode 100644 net/addrmanager.go delete mode 100644 net/connmanager.go delete mode 100644 net/knowaddress.go delete mode 100644 net/neighbors.go delete mode 100644 net/peer.go delete mode 100644 net/serverpeer.go rename sdk/spvpeer.go => peer/peer.go (81%) delete mode 100644 sdk/blockchain.go create mode 100644 sdk/interface.go delete mode 100644 sdk/spvserviceimpl.go delete mode 100644 sdk/syncmanager.go rename spvwallet/{cli => client}/account/account.go (99%) rename spvwallet/{cli => client}/common.go (91%) create mode 100644 spvwallet/client/database/database.go create mode 100644 spvwallet/client/interface.go rename spvwallet/{ => client}/keystore.go (100%) rename spvwallet/{ => client}/keystore_file.go (100%) rename spvwallet/{cli => client}/transaction/transaction.go (94%) rename spvwallet/{cli => client}/wallet/wallet.go (98%) delete mode 100644 spvwallet/database.go delete mode 100644 spvwallet/db/chain.go delete mode 100644 spvwallet/db/datastore.go delete mode 100644 spvwallet/spvwallet.go rename spvwallet/{db/headers.go => store/headers/database.go} (58%) rename spvwallet/{db/addrsdb.go => store/sqlite/addrs.go} (71%) rename spvwallet/{db/sqlitedb.go => store/sqlite/database.go} (55%) create mode 100644 spvwallet/store/sqlite/interface.go create mode 100644 spvwallet/store/sqlite/state.go rename spvwallet/{db/stxosdb.go => store/sqlite/stxos.go} (76%) rename spvwallet/{db/txsdb.go => store/sqlite/txs.go} (64%) rename spvwallet/{db/utxosdb.go => store/sqlite/utxos.go} (74%) rename spvwallet/{db => sutil}/addr.go (98%) rename spvwallet/{db => sutil}/stxo.go (98%) rename spvwallet/{db => sutil}/tx.go (98%) rename spvwallet/{db => sutil}/utxo.go (99%) delete mode 100644 store/headerstore.go rename store/storeheader.go => util/header.go (67%) diff --git a/blockchain/chain.go b/blockchain/chain.go new file mode 100644 index 0000000..016b486 --- /dev/null +++ b/blockchain/chain.go @@ -0,0 +1,277 @@ +package blockchain + +import ( + "errors" + "github.com/elastos/Elastos.ELA.SPV/database" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA/core" + "math/big" + "sync" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +const ( + MaxBlockLocatorHashes = 100 +) + +var zeroHash = common.Uint256{} + +var OrphanBlockError = errors.New("block does not extend any known blocks") + +/* +BlockChain is the database of blocks, also when a new transaction or block commit, +BlockChain will verify them with stored blocks. +*/ +type BlockChain struct { + lock sync.RWMutex + db database.ChainStore +} + +// NewBlockChain returns a new BlockChain instance. +func New(foundation string, db database.ChainStore) (*BlockChain, error) { + chain := &BlockChain{db: db} + + // Init genesis header + _, err := chain.db.Headers().GetBest() + if err != nil { + var err error + var foundationAddress *common.Uint168 + if len(foundation) == 34 { + foundationAddress, err = common.Uint168FromAddress(foundation) + } else { + foundationAddress, err = common.Uint168FromAddress("8VYXVxKKSAxkmRrfmGpQR2Kc66XhG6m3ta") + } + if err != nil { + return nil, errors.New("parse foundation address failed") + } + genesisHeader := GenesisHeader(foundationAddress) + storeHeader := &util.Header{Header: genesisHeader, TotalWork: new(big.Int)} + chain.db.Headers().Put(storeHeader, true) + } + + return chain, nil +} + +func (b *BlockChain) CommitTx(tx *core.Transaction) (bool, error) { + return b.db.StoreTx(&util.Tx{Transaction: tx, Height: 0}) +} + +func (b *BlockChain) CommitBlock(block *util.Block) (newTip, reorg bool, newHeight, fps uint32, err error) { + b.lock.Lock() + defer b.lock.Unlock() + newTip = false + reorg = false + var header = block.Header + var commonAncestor *util.Header + // Fetch our current best header from the db + bestHeader, err := b.db.Headers().GetBest() + if err != nil { + return false, false, 0, 0, err + } + tipHash := bestHeader.Hash() + var parentHeader *util.Header + + // If the tip is also the parent of this header, then we can save a database read by skipping + // the lookup of the parent header. Otherwise (ophan?) we need to fetch the parent. + if block.Previous.IsEqual(tipHash) { + parentHeader = bestHeader + } else { + parentHeader, err = b.db.Headers().GetPrevious(block.Header) + if err != nil { + return false, false, 0, 0, OrphanBlockError + } + } + valid := b.checkHeader(header, parentHeader) + if !valid { + return false, false, 0, 0, nil + } + // If this block is already the tip, return + headerHash := header.Hash() + if tipHash.IsEqual(headerHash) { + return false, false, 0, 0, nil + } + // Add the work of this header to the total work stored at the previous header + cumulativeWork := new(big.Int).Add(parentHeader.TotalWork, CalcWork(header.Bits)) + + // If the cumulative work is greater than the total work of our best header + // then we have a new best header. Update the chain tip and check for a reorg. + if cumulativeWork.Cmp(bestHeader.TotalWork) > 0 { + newTip = true + + // If this header is not extending the previous best header then we have a reorg. + if !tipHash.IsEqual(parentHeader.Hash()) { + reorg = true + } + } + + // At this point, we have done header check, so store it into database. + newHeight = parentHeader.Height + 1 + header.Height = newHeight + header.TotalWork = cumulativeWork + fps, err = b.db.StoreBlock(block, newTip) + if err != nil { + return newTip, reorg, 0, 0, err + } + + // If not meet a reorg, just return. + if !reorg { + return newTip, reorg, newHeight, fps, nil + } + + // Find common ancestor of the fork chain, so we can rollback chain to the + // point where fork has happened. + commonAncestor, err = b.getCommonAncestor(header, bestHeader) + if err != nil { + // This should not happen, because we didn't store orphan blocks in + // database, all headers should be connected. + log.Errorf("Error calculating common ancestor: %s", err.Error()) + return newTip, reorg, 0, 0, err + } + + // Rollback block chain to fork point. + log.Infof("REORG!!! At block %d, Wiped out %d blocks", + bestHeader.Height, bestHeader.Height-commonAncestor.Height) + err = b.db.Rollback(commonAncestor) + if err != nil { + return newTip, reorg, 0, 0, err + } + return newTip, reorg, newHeight, fps, nil +} + +func (b *BlockChain) checkHeader(header *util.Header, prevHeader *util.Header) bool { + // Get hash of n-1 header + prevHash := prevHeader.Hash() + height := prevHeader.Height + + // Check if headers link together. That whole 'blockchain' thing. + if prevHash.IsEqual(header.Previous) == false { + log.Errorf("Headers %d and %d don't link.\n", height, height+1) + return false + } + + // Check if there's a valid proof of work. That whole "Bitcoin" thing. + if !checkProofOfWork(*header) { + log.Debugf("Block %d bad proof of work.\n", height+1) + return false + } + + return true // it must have worked if there's no errors and got to the end. +} + +// Returns last header before reorg point +func (b *BlockChain) getCommonAncestor(bestHeader, prevTip *util.Header) (*util.Header, error) { + var err error + rollback := func(parent *util.Header, n int) (*util.Header, error) { + for i := 0; i < n; i++ { + parent, err = b.db.Headers().GetPrevious(parent) + if err != nil { + return parent, err + } + } + return parent, nil + } + + majority := bestHeader + minority := prevTip + if bestHeader.Height > prevTip.Height { + majority, err = rollback(majority, int(bestHeader.Height-prevTip.Height)) + if err != nil { + return nil, err + } + } else if prevTip.Height > bestHeader.Height { + minority, err = rollback(minority, int(prevTip.Height-bestHeader.Height)) + if err != nil { + return nil, err + } + } + + for { + majorityHash := majority.Hash() + minorityHash := minority.Hash() + if majorityHash.IsEqual(minorityHash) { + return majority, nil + } + majority, err = b.db.Headers().GetPrevious(majority) + if err != nil { + return nil, err + } + minority, err = b.db.Headers().GetPrevious(minority) + if err != nil { + return nil, err + } + } +} + +// HaveBlock returns whether or not the chain instance has the block represented +// by the passed hash. This includes checking the various places a block can +// be like part of the main chain, on a side chain, or in the orphan pool. +// +// This function is safe for concurrent access. +func (b *BlockChain) HaveBlock(hash *common.Uint256) bool { + header, err := b.db.Headers().Get(hash) + return err != nil && header != nil +} + +// LatestBlockLocator returns a block locator for current last block, +// which is a array of block hashes stored in blockchain +func (b *BlockChain) LatestBlockLocator() []*common.Uint256 { + b.lock.RLock() + defer b.lock.RUnlock() + + var ret []*common.Uint256 + parent, err := b.db.Headers().GetBest() + if err != nil { // No headers stored return empty locator + return ret + } + + rollback := func(parent *util.Header, n int) (*util.Header, error) { + for i := 0; i < n; i++ { + parent, err = b.db.Headers().GetPrevious(parent) + if err != nil { + return parent, err + } + } + return parent, nil + } + + step := 1 + start := 0 + for { + if start >= 9 { + step *= 2 + start = 0 + } + hash := parent.Hash() + ret = append(ret, &hash) + if len(ret) >= MaxBlockLocatorHashes { + break + } + parent, err = rollback(parent, step) + if err != nil { + break + } + start += 1 + } + return ret +} + +// BestHeight return current best chain height. +func (b *BlockChain) BestHeight() uint32 { + best, err := b.db.Headers().GetBest() + if err != nil { + return 0 + } + return best.Height +} + +// Close the blockchain +func (b *BlockChain) Clear() error { + return b.db.Clear() +} + +// Close the blockchain +func (b *BlockChain) Close() error { + b.lock.Lock() + return b.db.Close() +} diff --git a/blockchain/difficulty.go b/blockchain/difficulty.go new file mode 100644 index 0000000..3b748e3 --- /dev/null +++ b/blockchain/difficulty.go @@ -0,0 +1,86 @@ +package blockchain + +import ( + "math/big" + + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" +) + +var PowLimit = new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 255), big.NewInt(1)) + +func CalcWork(bits uint32) *big.Int { + // Return a work value of zero if the passed difficulty bits represent + // a negative number. Note this should not happen in practice with valid + // blocks, but an invalid block could trigger it. + difficultyNum := CompactToBig(bits) + if difficultyNum.Sign() <= 0 { + return big.NewInt(0) + } + + // (1 << 256) / (difficultyNum + 1) + denominator := new(big.Int).Add(difficultyNum, big.NewInt(1)) + return new(big.Int).Div(new(big.Int).Lsh(big.NewInt(1), 256), denominator) +} + +func checkProofOfWork(header util.Header) bool { + // The target difficulty must be larger than zero. + target := CompactToBig(header.Bits) + if target.Sign() <= 0 { + return false + } + + // The target difficulty must be less than the maximum allowed. + if target.Cmp(PowLimit) > 0 { + return false + } + + // The block hash must be less than the claimed target. + hash := header.AuxPow.ParBlockHeader.Hash() + hashNum := HashToBig(&hash) + if hashNum.Cmp(target) > 0 { + return false + } + + return true +} + +func HashToBig(hash *common.Uint256) *big.Int { + // A Hash is in little-endian, but the big package wants the bytes in + // big-endian, so reverse them. + buf := *hash + blen := len(buf) + for i := 0; i < blen/2; i++ { + buf[i], buf[blen-1-i] = buf[blen-1-i], buf[i] + } + + return new(big.Int).SetBytes(buf[:]) +} + +func CompactToBig(compact uint32) *big.Int { + // Extract the mantissa, sign bit, and exponent. + mantissa := compact & 0x007fffff + isNegative := compact&0x00800000 != 0 + exponent := uint(compact >> 24) + + // Since the base for the exponent is 256, the exponent can be treated + // as the number of bytes to represent the full 256-bit number. So, + // treat the exponent as the number of bytes and shift the mantissa + // right or left accordingly. This is equivalent to: + // N = mantissa * 256^(exponent-3) + var bn *big.Int + if exponent <= 3 { + mantissa >>= 8 * (3 - exponent) + bn = big.NewInt(int64(mantissa)) + } else { + bn = big.NewInt(int64(mantissa)) + bn.Lsh(bn, 8*(exponent-3)) + } + + // Make it negative if the sign bit is set. + if isNegative { + bn = bn.Neg(bn) + } + + return bn +} diff --git a/blockchain/genesis.go b/blockchain/genesis.go new file mode 100644 index 0000000..9407e01 --- /dev/null +++ b/blockchain/genesis.go @@ -0,0 +1,85 @@ +package blockchain + +import ( + "time" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/crypto" + "github.com/elastos/Elastos.ELA/core" +) + +// GenesisHeader creates a specific genesis header by the given +// foundation address. +func GenesisHeader(foundation *common.Uint168) *core.Header { + // Genesis time + genesisTime := time.Date(2017, time.December, 22, 10, 0, 0, 0, time.UTC) + + // header + header := core.Header{ + Version: core.BlockVersion, + Previous: common.EmptyHash, + MerkleRoot: common.EmptyHash, + Timestamp: uint32(genesisTime.Unix()), + Bits: 0x1d03ffff, + Nonce: core.GenesisNonce, + Height: uint32(0), + } + + // ELA coin + elaCoin := &core.Transaction{ + TxType: core.RegisterAsset, + PayloadVersion: 0, + Payload: &core.PayloadRegisterAsset{ + Asset: core.Asset{ + Name: "ELA", + Precision: 0x08, + AssetType: 0x00, + }, + Amount: 0 * 100000000, + Controller: common.Uint168{}, + }, + Attributes: []*core.Attribute{}, + Inputs: []*core.Input{}, + Outputs: []*core.Output{}, + Programs: []*core.Program{}, + } + + coinBase := &core.Transaction{ + TxType: core.CoinBase, + PayloadVersion: core.PayloadCoinBaseVersion, + Payload: new(core.PayloadCoinBase), + Inputs: []*core.Input{ + { + Previous: core.OutPoint{ + TxID: common.EmptyHash, + Index: 0x0000, + }, + Sequence: 0x00000000, + }, + }, + Attributes: []*core.Attribute{}, + LockTime: 0, + Programs: []*core.Program{}, + } + + coinBase.Outputs = []*core.Output{ + { + AssetID: elaCoin.Hash(), + Value: 3300 * 10000 * 100000000, + ProgramHash: *foundation, + }, + } + + nonce := []byte{0x4d, 0x65, 0x82, 0x21, 0x07, 0xfc, 0xfd, 0x52} + txAttr := core.NewAttribute(core.Nonce, nonce) + coinBase.Attributes = append(coinBase.Attributes, &txAttr) + + transactions := []*core.Transaction{coinBase, elaCoin} + hashes := make([]common.Uint256, 0, len(transactions)) + for _, tx := range transactions { + hashes = append(hashes, tx.Hash()) + } + header.MerkleRoot, _ = crypto.ComputeRoot(hashes) + + return &header +} diff --git a/blockchain/log.go b/blockchain/log.go new file mode 100644 index 0000000..e0edc6e --- /dev/null +++ b/blockchain/log.go @@ -0,0 +1,28 @@ +package blockchain + +import ( + "github.com/elastos/Elastos.ELA.Utility/elalog" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log elalog.Logger + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = elalog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using elalog. +func UseLogger(logger elalog.Logger) { + log = logger +} diff --git a/database/headers.go b/database/headers.go new file mode 100644 index 0000000..c02bbd4 --- /dev/null +++ b/database/headers.go @@ -0,0 +1,27 @@ +package database + +import ( + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +type Headers interface { + // Extend from DB interface + DB + + // Save a header to database + PutHeader(header *util.Header, newTip bool) error + + // Get previous block of the given header + GetPrevious(header *util.Header) (*util.Header, error) + + // Get full header with it's hash + GetHeader(hash *common.Uint256) (*util.Header, error) + + // Get the header on chain tip + GetBestHeader() (*util.Header, error) + + // DelHeader delete a header save in database by it's hash. + DelHeader(hash *common.Uint256) error +} diff --git a/net/addrmanager.go b/net/addrmanager.go deleted file mode 100644 index 6b695c4..0000000 --- a/net/addrmanager.go +++ /dev/null @@ -1,184 +0,0 @@ -package net - -import ( - "sync" - "time" - - "github.com/elastos/Elastos.ELA.Utility/p2p" -) - -const ( - // needAddressThreshold is the number of addresses under which the - // address manager will claim to need more addresses. - needAddressThreshold = 100 - // addressListMonitorDuration is the time gap to check and remove - // any bad addresses from address list - addressListMonitorDuration = time.Hour * 12 -) - -type AddrManager struct { - mutex sync.RWMutex - minOutbound int - addresses *addrList - connected *addrList -} - -type addrList struct { - list map[string]*knownAddress -} - -func newAddressesList() *addrList { - return &addrList{ - list: make(map[string]*knownAddress), - } -} - -func (a *addrList) put(key string, value *knownAddress) { - a.list[key] = value -} - -func (a *addrList) get(key string) *knownAddress { - return a.list[key] -} - -func (a *addrList) exist(key string) bool { - _, ok := a.list[key] - return ok -} - -func (a *addrList) del(key string) { - delete(a.list, key) -} - -func (a *addrList) size() int { - return len(a.list) -} - -func newAddrManager(minOutbound int) *AddrManager { - am := new(AddrManager) - am.minOutbound = minOutbound - am.addresses = newAddressesList() - am.connected = newAddressesList() - return am -} - -func (am *AddrManager) NeedMoreAddresses() bool { - am.mutex.RLock() - defer am.mutex.RUnlock() - return am.addresses.size() < needAddressThreshold -} - -func (am *AddrManager) GetOutboundAddresses() []p2p.NetAddress { - am.mutex.RLock() - defer am.mutex.RUnlock() - - var addrs []p2p.NetAddress - for _, addr := range SortAddressMap(am.addresses.list) { - address := addr.String() - // Skip connected address - if am.connected.exist(address) { - continue - } - addr.increaseAttempts() - addr.updateLastAttempt() - // Skip bad address - if addr.isBad() { - continue - } - addrs = append(addrs, addr.NetAddress) - if len(addrs) >= am.minOutbound { - break - } - } - return addrs -} - -func (am *AddrManager) RandGetAddresses() []p2p.NetAddress { - am.mutex.RLock() - defer am.mutex.RUnlock() - - var addrs []p2p.NetAddress - for _, addr := range am.addresses.list { - if addr.isBad() { - continue - } - addrs = append(addrs, addr.NetAddress) - if len(addrs) >= am.minOutbound*2 { - break - } - } - return addrs -} - -func (am *AddrManager) AddressConnected(na *p2p.NetAddress) { - am.mutex.Lock() - defer am.mutex.Unlock() - - addr := na.String() - // Try add to address list - am.addOrUpdateAddress(na) - if !am.connected.exist(addr) { - ka := am.addresses.get(addr) - ka.SaveAddr(na) - am.connected.put(addr, ka) - } -} - -func (am *AddrManager) AddressDisconnect(na *p2p.NetAddress) { - am.mutex.Lock() - defer am.mutex.Unlock() - - addr := na.String() - // Update disconnect time - ka := am.addresses.get(addr) - ka.updateLastDisconnect() - // Delete from connected list - am.connected.del(addr) -} - -func (am *AddrManager) AddOrUpdateAddress(na *p2p.NetAddress) { - am.mutex.Lock() - defer am.mutex.Unlock() - - am.addOrUpdateAddress(na) -} - -func (am *AddrManager) addOrUpdateAddress(na *p2p.NetAddress) { - addr := na.String() - // Update already known address - ka := am.addresses.get(addr) - if ka == nil { - ka := new(knownAddress) - ka.SaveAddr(na) - // Add to address list - am.addresses.put(addr, ka) - } else { - ka.SaveAddr(na) - } -} - -func (am *AddrManager) KnowAddresses() []p2p.NetAddress { - am.mutex.RLock() - defer am.mutex.RUnlock() - - nas := make([]p2p.NetAddress, 0, am.addresses.size()) - for _, ka := range am.addresses.list { - nas = append(nas, ka.NetAddress) - } - return nas -} - -func (am *AddrManager) monitorAddresses() { - ticker := time.NewTicker(addressListMonitorDuration) - defer ticker.Stop() - - for range ticker.C { - am.mutex.Lock() - for addr, ka := range am.addresses.list { - if ka.isBad() { - delete(am.addresses.list, addr) - } - } - am.mutex.Unlock() - } -} diff --git a/net/connmanager.go b/net/connmanager.go deleted file mode 100644 index 16f42e1..0000000 --- a/net/connmanager.go +++ /dev/null @@ -1,130 +0,0 @@ -package net - -import ( - "net" - "sync" - "time" - - "fmt" - "github.com/elastos/Elastos.ELA.SPV/log" -) - -const ( - DialTimeout = time.Second * 10 - HandshakeTimeout = time.Second * 10 -) - -type ConnectionListener interface { -} - -type ConnManager struct { - port uint16 - maxConnections int - - mutex *sync.RWMutex - connections map[string]net.Conn - - OnConnection func(conn net.Conn, inbound bool) -} - -func newConnManager(port uint16, maxConnections int) *ConnManager { - cm := new(ConnManager) - cm.port = port - cm.maxConnections = maxConnections - cm.mutex = new(sync.RWMutex) - cm.connections = make(map[string]net.Conn) - return cm -} - -func (cm *ConnManager) resolveAddr(addr string) (string, error) { - tcpAddr, err := net.ResolveTCPAddr("tcp", addr) - if err != nil { - log.Debugf("Can not resolve address %s", addr) - return addr, err - } - return tcpAddr.String(), nil -} - -func (cm *ConnManager) Connect(addr string) { - tcpAddr, err := cm.resolveAddr(addr) - if err != nil { - return - } - - if cm.IsConnected(tcpAddr) { - log.Debugf("Seed %s already connected", addr) - return - } - - conn, err := net.DialTimeout("tcp", tcpAddr, DialTimeout) - if err != nil { - log.Error("Connect to addr ", addr, " failed, err", err) - return - } - - // Callback outbound connection - cm.OnConnection(conn, false) -} - -func (cm *ConnManager) IsConnected(addr string) bool { - cm.mutex.RLock() - defer cm.mutex.RUnlock() - _, ok := cm.connections[addr] - return ok -} - -func (cm *ConnManager) PeerConnected(addr string, conn net.Conn) { - cm.mutex.Lock() - defer cm.mutex.Unlock() - // Add to connection list - cm.connections[addr] = conn -} - -func (cm *ConnManager) PeerDisconnected(addr string) { - cm.mutex.Lock() - defer cm.mutex.Unlock() - - if _, ok := cm.connections[addr]; ok { - delete(cm.connections, addr) - } -} - -func (cm *ConnManager) listenConnection() { - listener, err := net.Listen("tcp", fmt.Sprint(":", cm.port)) - if err != nil { - fmt.Println("Start peer listening err, ", err.Error()) - return - } - defer listener.Close() - - for { - conn, err := listener.Accept() - if err != nil { - fmt.Println("Error accepting ", err.Error()) - continue - } - log.Debugf("New connection accepted, remote: %s local: %s", conn.RemoteAddr(), conn.LocalAddr()) - - // Callback inbound connection - cm.OnConnection(conn, true) - } -} - -func (cm *ConnManager) monitorConnections() { - ticker := time.NewTicker(InfoUpdateDuration) - for range ticker.C { - cm.mutex.Lock() - conns := len(cm.connections) - if conns > cm.maxConnections { - // Random close connections - for _, conn := range cm.connections { - conn.Close() - conns-- - if conns <= cm.maxConnections { - break - } - } - } - cm.mutex.Unlock() - } -} diff --git a/net/knowaddress.go b/net/knowaddress.go deleted file mode 100644 index aba42a0..0000000 --- a/net/knowaddress.go +++ /dev/null @@ -1,107 +0,0 @@ -package net - -import ( - "time" - "sort" - - "github.com/elastos/Elastos.ELA.Utility/p2p" -) - -const ( - // numMissingDays is the number of days before which we assume an - // address has vanished if we have not seen it announced in that long. - numMissingDays = 30 -) - -type knownAddress struct { - p2p.NetAddress - lastAttempt time.Time - lastDisconnect time.Time - attempts uint32 -} - -func (ka *knownAddress) LastAttempt() time.Time { - return ka.lastAttempt -} - -func (ka *knownAddress) increaseAttempts() { - ka.attempts++ -} - -func (ka *knownAddress) updateLastAttempt() { - // set last tried time to now - ka.lastAttempt = time.Now() -} - -func (ka *knownAddress) updateLastDisconnect() { - // set last disconnect time to now - ka.lastDisconnect = time.Now() -} - -// chance returns the selection probability for a known address. The priority -// depends upon how recently the address has been seen, how recently it was last -// attempted and how often attempts to connect to it have failed. -func (ka *knownAddress) chance() float64 { - lastAttempt := time.Now().Sub(ka.lastAttempt) - - if lastAttempt < 0 { - lastAttempt = 0 - } - - c := 1.0 - - // Very recent attempts are less likely to be retried. - if lastAttempt < 10*time.Minute { - c *= 0.01 - } - - // Failed attempts deprioritise. - for i := ka.attempts; i > 0; i-- { - c /= 1.5 - } - - return c -} - -// isBad returns true if the address in question has not been seen in over a month -func (ka *knownAddress) isBad() bool { - // Over a month old? - if ka.Time < time.Now().Add(-1 * numMissingDays * time.Hour * 24).UnixNano() { - return true - } - - return false -} - -func (ka *knownAddress) SaveAddr(na *p2p.NetAddress) { - ka.ID = na.ID - ka.IP = na.IP - ka.Port = na.Port - ka.Services = na.Services - ka.Time = na.Time -} - -type OrderByChance []*knownAddress - -// Len is the number of elements in the collection. -func (c OrderByChance) Len() int { return len(c) } - -// Less reports whether the element with -// index i should sort before the element with index j. -func (c OrderByChance) Less(i, j int) bool { return c[i].chance() > c[j].chance() } - -// Swap swaps the elements with indexes i and j. -func (c OrderByChance) Swap(i, j int) { c[i], c[j] = c[j], c[i] } - -func SortAddressMap(addrMap map[string]*knownAddress) []*knownAddress { - var addrList []*knownAddress - for _, addr := range addrMap { - addrList = append(addrList, addr) - } - return SortAddressList(addrList) -} - -func SortAddressList(addrList []*knownAddress) []*knownAddress { - sort.Sort(OrderByChance(addrList)) - return addrList -} diff --git a/net/neighbors.go b/net/neighbors.go deleted file mode 100644 index fafa7d1..0000000 --- a/net/neighbors.go +++ /dev/null @@ -1,72 +0,0 @@ -package net - -import ( - "sync" -) - -type neighbors struct { - sync.Mutex - list map[uint64]*Peer -} - -func (ns *neighbors) Init() { - ns.list = make(map[uint64]*Peer) -} - -func (ns *neighbors) AddNeighbor(peer *Peer) { - ns.Lock() - defer ns.Unlock() - - ns.list[peer.ID()] = peer -} - -func (ns *neighbors) DelNeighbor(id uint64) (*Peer, bool) { - ns.Lock() - defer ns.Unlock() - - peer, ok := ns.list[id] - delete(ns.list, id) - - return peer, ok -} - -func (ns *neighbors) GetNeighborCount() (count int) { - ns.Lock() - defer ns.Unlock() - - for _, n := range ns.list { - if !n.Connected() { - continue - } - count++ - } - return count -} - -func (ns *neighbors) GetNeighborPeers() []*Peer { - ns.Lock() - defer ns.Unlock() - - peers := make([]*Peer, 0, len(ns.list)) - for _, peer := range ns.list { - // Skip disconnected peer - if !peer.Connected() { - continue - } - - peers = append(peers, peer) - } - return peers -} - -func (ns *neighbors) IsNeighborPeer(id uint64) bool { - ns.Lock() - defer ns.Unlock() - - peer, ok := ns.list[id] - if !ok { - return false - } - - return peer.Connected() -} diff --git a/net/peer.go b/net/peer.go deleted file mode 100644 index 5b8e5f6..0000000 --- a/net/peer.go +++ /dev/null @@ -1,548 +0,0 @@ -package net - -import ( - "container/list" - "fmt" - "io" - "net" - "strings" - "sync/atomic" - "time" - - "github.com/elastos/Elastos.ELA.SPV/log" - - "github.com/elastos/Elastos.ELA.Utility/p2p" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA.Utility/p2p/rw" -) - -const ( - // outputBufferSize is the number of elements the output channels use. - outputBufferSize = 50 - - // idleTimeout is the duration of inactivity before we time out a peer. - idleTimeout = 2 * time.Minute - - // pingInterval is the interval of time to wait in between sending ping - // messages. - pingInterval = 30 * time.Second -) - -// outMsg is used to house a message to be sent along with a channel to signal -// when the message has been sent (or won't be sent due to things such as -// shutdown) -type outMsg struct { - msg p2p.Message - doneChan chan<- struct{} -} - -type PeerConfig struct { - PingNonce func() uint32 - PongNonce func() uint32 - OnVerAck func(peer *Peer) - OnGetAddr func(peer *Peer) - OnAddr func(peer *Peer, addr *msg.Addr) - OnPing func(peer *Peer, ping *msg.Ping) - OnPong func(peer *Peer, pong *msg.Pong) - HandleMessage func(peer *Peer, msg p2p.Message) -} - -type Peer struct { - id uint64 - version uint32 - services uint64 - ip16 [16]byte - port uint16 - lastActive time.Time - height uint64 - relay uint8 // 1 for true 0 for false - - disconnect int32 - conn net.Conn - - rw rw.MessageRW - config PeerConfig - outputQueue chan outMsg - sendQueue chan outMsg - sendDoneQueue chan struct{} - inQuit chan struct{} - queueQuit chan struct{} - outQuit chan struct{} - quit chan struct{} -} - -func (p *Peer) String() string { - return fmt.Sprint( - "ID:", p.id, - ", Version:", p.version, - ", Services:", p.services, - ", Port:", p.port, - ", LastActive:", p.lastActive, - ", Height:", p.height, - ", Relay:", p.relay, - ", Addr:", p.Addr().String()) -} - -func (p *Peer) ID() uint64 { - return p.id -} - -func (p *Peer) SetID(id uint64) { - p.id = id -} - -func (p *Peer) Version() uint32 { - return p.version -} - -func (p *Peer) SetVersion(version uint32) { - p.version = version -} - -func (p *Peer) Services() uint64 { - return p.services -} - -func (p *Peer) SetServices(services uint64) { - p.services = services -} - -func (p *Peer) IP16() [16]byte { - return p.ip16 -} - -func (p *Peer) Port() uint16 { - return p.port -} - -func (p *Peer) SetPort(port uint16) { - p.port = port -} - -func (p *Peer) LastActive() time.Time { - return p.lastActive -} - -func (p *Peer) Addr() *p2p.NetAddress { - return p2p.NewNetAddress(p.services, p.ip16, p.port, p.id) -} - -func (p *Peer) Relay() uint8 { - return p.relay -} - -func (p *Peer) SetRelay(relay uint8) { - p.relay = relay -} - -func (p *Peer) SetInfo(msg *msg.Version) { - p.id = msg.Nonce - p.port = msg.Port - p.version = msg.Version - p.services = msg.Services - p.lastActive = time.Now() - p.height = msg.Height - p.relay = msg.Relay -} - -func (p *Peer) SetHeight(height uint64) { - p.height = height -} - -func (p *Peer) Height() uint64 { - return p.height -} - -// Connected returns whether or not the peer is currently connected. -// -// This function is safe for concurrent access. -func (p *Peer) Connected() bool { - return atomic.LoadInt32(&p.disconnect) == 0 -} - -// Disconnect disconnects the peer by closing the connection. Calling this -// function when the peer is already disconnected or in the process of -// disconnecting will have no effect. -func (p *Peer) Disconnect() { - // Return if peer already disconnected - if atomic.AddInt32(&p.disconnect, 1) != 1 { - return - } - - p.conn.Close() - close(p.quit) -} - -func (p *Peer) QuitChan() chan struct{} { - return p.quit -} - -// shouldHandleReadError returns whether or not the passed error, which is -// expected to have come from reading from the remote peer in the inHandler, -// should be logged and responded to with a reject message. -func (p *Peer) shouldHandleReadError(err error) bool { - // No logging or reject message when the peer is being forcibly - // disconnected. - if atomic.LoadInt32(&p.disconnect) != 0 { - return false - } - - // No logging or reject message when the remote peer has been - // disconnected. - if err == io.EOF { - return false - } - if opErr, ok := err.(*net.OpError); ok && !opErr.Temporary() { - return false - } - - return true -} - -func (p *Peer) inHandler() { - // The timer is stopped when a new message is received and reset after it - // is processed. - idleTimer := time.AfterFunc(idleTimeout, func() { - log.Warnf("Peer %s no answer for %s -- disconnecting", p, idleTimeout) - p.Disconnect() - }) - -out: - for atomic.LoadInt32(&p.disconnect) == 0 { - // Read a message and stop the idle timer as soon as the read - // is done. The timer is reset below for the next iteration if - // needed. - rmsg, err := p.readMessage() - idleTimer.Stop() - if err != nil { - // Only log the error and send reject message if the - // local peer is not forcibly disconnecting and the - // remote peer has not disconnected. - if p.shouldHandleReadError(err) { - errMsg := fmt.Sprintf("Can't read message from %s: %v", p, err) - if err != io.ErrUnexpectedEOF { - log.Errorf(errMsg) - } - - // Push a reject message for the malformed message and wait for - // the message to be sent before disconnecting. - // - // NOTE: Ideally this would include the command in the header if - // at least that much of the message was valid, but that is not - // currently exposed by wire, so just used malformed for the - // command. - rejectMsg := msg.NewReject("malformed", msg.RejectMalformed, errMsg) - // Send the message and block until it has been sent before returning. - doneChan := make(chan struct{}, 1) - p.QueueMessage(rejectMsg, doneChan) - <-doneChan - } - break out - } - log.Debugf("-----> inHandler [%s] from [0x%x]", rmsg.CMD(), p.id) - - // Handle each message. - switch m := rmsg.(type) { - case *msg.VerAck: - if p.config.OnVerAck != nil { - p.config.OnVerAck(p) - } - - case *msg.GetAddr: - if p.config.OnGetAddr != nil { - p.config.OnGetAddr(p) - } - - case *msg.Addr: - if p.config.OnAddr != nil { - p.config.OnAddr(p, m) - } - - case *msg.Ping: - if p.config.PongNonce != nil { - p.QueueMessage(msg.NewPong(p.config.PongNonce()), nil) - } - - if p.config.OnPing != nil { - p.config.OnPing(p, m) - } - - case *msg.Pong: - if p.config.OnPong != nil { - p.config.OnPong(p, m) - } - - default: - if p.config.HandleMessage != nil { - p.config.HandleMessage(p, rmsg) - } - } - - // A message was received so reset the idle timer. - idleTimer.Reset(idleTimeout) - } - - // Ensure the idle timer is stopped to avoid leaking the resource. - idleTimer.Stop() - - // Ensure connection is closed. - p.Disconnect() - - close(p.inQuit) -} - -func (p *Peer) queueHandler() { - pendingMsgs := list.New() - - // We keep the waiting flag so that we know if we have a message queued - // to the outHandler or not. We could use the presence of a head of - // the list for this but then we have rather racy concerns about whether - // it has gotten it at cleanup time - and thus who sends on the - // message's done channel. To avoid such confusion we keep a different - // flag and pendingMsgs only contains messages that we have not yet - // passed to outHandler. - waiting := false - - // To avoid duplication below. - queuePacket := func(msg outMsg, list *list.List, waiting bool) bool { - if !waiting { - p.sendQueue <- msg - } else { - list.PushBack(msg) - } - // we are always waiting now. - return true - } -out: - for { - select { - case msg := <-p.outputQueue: - waiting = queuePacket(msg, pendingMsgs, waiting) - - // This channel is notified when a message has been sent across - // the network socket. - case <-p.sendDoneQueue: - // No longer waiting if there are no more messages - // in the pending messages queue. - next := pendingMsgs.Front() - if next == nil { - waiting = false - continue - } - - // Notify the outHandler about the next item to - // asynchronously send. - val := pendingMsgs.Remove(next) - p.sendQueue <- val.(outMsg) - - case <-p.quit: - break out - } - } - - // Drain any wait channels before we go away so we don't leave something - // waiting for us. - for e := pendingMsgs.Front(); e != nil; e = pendingMsgs.Front() { - val := pendingMsgs.Remove(e) - msg := val.(outMsg) - if msg.doneChan != nil { - msg.doneChan <- struct{}{} - } - } -cleanup: - for { - select { - case msg := <-p.outputQueue: - if msg.doneChan != nil { - msg.doneChan <- struct{}{} - } - default: - break cleanup - } - } - close(p.queueQuit) - log.Tracef("Peer queue handler done for %s", p) -} - -func (p *Peer) outHandler() { -out: - for { - select { - case msg := <-p.sendQueue: - err := p.writeMessage(msg.msg) - if err != nil { - p.Disconnect() - if msg.doneChan != nil { - msg.doneChan <- struct{}{} - } - continue - } - log.Debugf("-----> outHandler [%s] to [0x%x]", msg.msg.CMD(), p.id) - - if msg.doneChan != nil { - msg.doneChan <- struct{}{} - } - p.sendDoneQueue <- struct{}{} - - case <-p.quit: - break out - } - } - - <-p.queueQuit - - // Drain any wait channels before we go away so we don't leave something - // waiting for us. We have waited on queueQuit and thus we can be sure - // that we will not miss anything sent on sendQueue. -cleanup: - for { - select { - case msg := <-p.sendQueue: - if msg.doneChan != nil { - msg.doneChan <- struct{}{} - } - // no need to send on sendDoneQueue since queueHandler - // has been waited on and already exited. - default: - break cleanup - } - } - close(p.outQuit) - log.Tracef("Peer output handler done for %s", p) -} - -// pingHandler periodically pings the peer. It must be run as a goroutine. -func (p *Peer) pingHandler() { - pingTicker := time.NewTicker(pingInterval) - defer pingTicker.Stop() - -out: - for { - select { - case <-pingTicker.C: - p.QueueMessage(msg.NewPing(p.config.PingNonce()), nil) - - case <-p.quit: - break out - } - } -} - -func (p *Peer) readMessage() (p2p.Message, error) { - return p.rw.ReadMessage(p.conn) -} - -func (p *Peer) writeMessage(msg p2p.Message) error { - // Don't do anything if we're disconnecting. - if atomic.LoadInt32(&p.disconnect) != 0 { - return nil - } - - return p.rw.WriteMessage(p.conn, msg) -} - -func (p *Peer) QueueMessage(msg p2p.Message, doneChan chan<- struct{}) { - if atomic.LoadInt32(&p.disconnect) != 0 { - if doneChan != nil { - go func() { - doneChan <- struct{}{} - }() - } - return - } - p.outputQueue <- outMsg{msg: msg, doneChan: doneChan} -} - -func (p *Peer) start() { - go p.inHandler() - go p.queueHandler() - go p.outHandler() - go p.pingHandler() -} - -func (p *Peer) NewVersionMsg() *msg.Version { - version := new(msg.Version) - version.Version = p.Version() - version.Services = p.Services() - version.TimeStamp = uint32(time.Now().UnixNano()) - version.Port = p.Port() - version.Nonce = p.ID() - version.Height = p.Height() - version.Relay = p.Relay() - return version -} - -func (p *Peer) SetPeerConfig(config PeerConfig) { - // Set PingNonce method - if config.PingNonce != nil { - p.config.PingNonce = config.PingNonce - } - - // Set OnVerAck method - if config.OnVerAck != nil { - p.config.OnVerAck = config.OnVerAck - } - - // Set OnGetAddr method - if config.OnGetAddr != nil { - p.config.OnGetAddr = config.OnGetAddr - } - - // Set OnGetAddr method - if config.OnAddr != nil { - p.config.OnAddr = config.OnAddr - } - - // Set OnPing method - if config.OnPing != nil { - p.config.OnPing = config.OnPing - } - - // Set OnPong method - if config.OnPong != nil { - p.config.OnPong = config.OnPong - } - - if config.HandleMessage != nil { - if p.config.HandleMessage == nil { - // Set message handler - p.config.HandleMessage = config.HandleMessage - - } else { - // Upgrade peer message handler - previousHandler := p.config.HandleMessage - p.config.HandleMessage = func(peer *Peer, msg p2p.Message) { - previousHandler(peer, msg) - config.HandleMessage(peer, msg) - } - } - } -} - -func (p *Peer) SetMessageConfig(config rw.MessageConfig) { - // Set rw config - p.rw.SetConfig(config) -} - -func NewPeer(magic uint32, conn net.Conn) *Peer { - p := Peer{ - conn: conn, - rw: rw.GetMesssageRW(magic), - outputQueue: make(chan outMsg, outputBufferSize), - sendQueue: make(chan outMsg, 1), // nonblocking sync - sendDoneQueue: make(chan struct{}, 1), // nonblocking sync - inQuit: make(chan struct{}), - queueQuit: make(chan struct{}), - outQuit: make(chan struct{}), - quit: make(chan struct{}), - } - - copy(p.ip16[:], getIp(conn)) - return &p -} - -func getIp(conn net.Conn) []byte { - addr := conn.RemoteAddr().String() - portIndex := strings.LastIndex(addr, ":") - return net.ParseIP(string([]byte(addr)[:portIndex])).To16() -} diff --git a/net/serverpeer.go b/net/serverpeer.go deleted file mode 100644 index f747d0e..0000000 --- a/net/serverpeer.go +++ /dev/null @@ -1,295 +0,0 @@ -package net - -import ( - "errors" - "github.com/elastos/Elastos.ELA.SPV/log" - "time" - - "github.com/elastos/Elastos.ELA.Utility/p2p" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "net" -) - -const ( - MinConnections = 3 - InfoUpdateDuration = time.Second * 5 -) - -// P2P network config -type ServerPeerConfig struct { - Magic uint32 - Version uint32 - PeerId uint64 - Port uint16 - Seeds []string - MinOutbound int - MaxConnections int -} - -// Handle the message creation, allocation etc. -type PeerManageConfig struct { - // A handshake message received - OnHandshake func(v *msg.Version) error - - // VerAck message received from a connected peer - // which means the connected peer is established - OnPeerEstablish func(*Peer) -} - -type ServerPeer struct { - Peer - magic uint32 - seeds []string - neighbors - quitChan chan uint64 - am *AddrManager - cm *ConnManager - config PeerManageConfig -} - -func NewServerPeer(config ServerPeerConfig) *ServerPeer { - // Initiate ServerPeer - sp := new(ServerPeer) - sp.id = config.PeerId - sp.version = config.Version - sp.port = config.Port - sp.magic = config.Magic - sp.seeds = config.Seeds - sp.neighbors.Init() - sp.quitChan = make(chan uint64, 1) - sp.am = newAddrManager(config.MinOutbound) - sp.cm = newConnManager(sp.port, config.MaxConnections) - sp.cm.OnConnection = sp.OnConnection - return sp -} - -func (sp *ServerPeer) SetConfig(config PeerManageConfig) { - sp.config = config -} - -func (sp *ServerPeer) Start() { - log.Info("ServerPeer start") - go sp.keepConnections() - go sp.peerQuitHandler() - go sp.am.monitorAddresses() - go sp.cm.listenConnection() - go sp.cm.monitorConnections() -} - -func (sp *ServerPeer) OnConnection(conn net.Conn, inbound bool) { - // Start handshake - doneChan := make(chan struct{}) - go func() { - sp.handshake(NewPeer(sp.magic, conn), inbound, doneChan) - }() - <-doneChan -} - -func (sp *ServerPeer) readVersionMsg(peer *Peer) error { - message, err := peer.readMessage() - if err != nil { - return err - } - - version, ok := message.(*msg.Version) - if !ok { - errMsg := "A version message must precede all others" - log.Error(errMsg) - - reject := msg.NewReject(message.CMD(), msg.RejectMalformed, errMsg) - return peer.writeMessage(reject) - } - - if err := sp.config.OnHandshake(version); err != nil { - return err - } - - // Check if handshake with itself - if version.Nonce == sp.ID() { - peer.Disconnect() - return errors.New("peer handshake with itself") - } - - // If peer already connected, disconnect previous peer - if oldPeer, ok := sp.DelNeighbor(version.Nonce); ok { - log.Warnf("Peer %d reconnect", version.Nonce) - oldPeer.Disconnect() - } - - // Set peer info with version message - peer.SetInfo(version) - - return nil -} - -func (sp *ServerPeer) inboundProtocol(peer *Peer) error { - if err := sp.readVersionMsg(peer); err != nil { - return err - } - - return peer.writeMessage(sp.NewVersionMsg()) -} - -func (sp *ServerPeer) outboundProtocol(peer *Peer) error { - if err := peer.writeMessage(sp.NewVersionMsg()); err != nil { - return err - } - - return sp.readVersionMsg(peer) -} - -func (sp *ServerPeer) handshake(peer *Peer, inbound bool, doneChan chan struct{}) { - errChan := make(chan error) - go func() { - if inbound { - errChan <- sp.inboundProtocol(peer) - } else { - errChan <- sp.outboundProtocol(peer) - } - }() - - select { - case err := <-errChan: - if err != nil { - return - } - - case <-time.After(HandshakeTimeout): - // Disconnect peer for handshake timeout - peer.Disconnect() - - // Notify handshake done - doneChan <- struct{}{} - return - } - - // Wait for peer quit - go func() { - select { - case <-peer.quit: - sp.quitChan <- peer.id - } - }() - - // Add peer to neighbor list - sp.AddToNeighbors(peer) - - // Notify handshake done - doneChan <- struct{}{} - - // Update peer's message config - peer.SetPeerConfig(sp.basePeerConfig()) - - // Start peer - peer.start() - - // Send our verack message now that the IO processing machinery has started. - peer.QueueMessage(new(msg.VerAck), nil) -} - -func (sp *ServerPeer) peerQuitHandler() { - for peerId := range sp.quitChan { - if peer, ok := sp.neighbors.DelNeighbor(peerId); ok { - log.Trace("ServerPeer peer disconnected:", peer) - na := peer.Addr() - addr := na.String() - sp.am.AddressDisconnect(na) - sp.cm.PeerDisconnected(addr) - } - } -} - -func (sp *ServerPeer) AddToNeighbors(peer *Peer) { - log.Trace("ServerPeer add connected peer:", peer) - // Add peer to list - sp.AddNeighbor(peer) - - // Mark addr as connected - addr := peer.Addr() - sp.am.AddressConnected(addr) - sp.cm.PeerConnected(addr.String(), peer.conn) -} - -func (sp *ServerPeer) KnownAddresses() []p2p.NetAddress { - return sp.am.KnowAddresses() -} - -func (sp *ServerPeer) keepConnections() { - for { - // connect seeds first - if sp.GetNeighborCount() < MinConnections { - for _, seed := range sp.seeds { - sp.cm.Connect(seed) - } - - } else if sp.GetNeighborCount() < sp.am.minOutbound { - for _, addr := range sp.am.GetOutboundAddresses() { - sp.cm.Connect(addr.String()) - } - } - - // request more addresses - if sp.am.NeedMoreAddresses() { - sp.Broadcast(new(msg.GetAddr)) - } - - time.Sleep(InfoUpdateDuration) - } -} - -func (sp *ServerPeer) basePeerConfig() PeerConfig { - return PeerConfig{ - OnVerAck: func(peer *Peer) { - // Notify peer establish - sp.config.OnPeerEstablish(peer) - }, - - OnGetAddr: func(peer *Peer) { - peer.QueueMessage(msg.NewAddr(sp.am.RandGetAddresses()), nil) - }, - - OnAddr: func(peer *Peer, m *msg.Addr) { - for _, addr := range m.AddrList { - // Skip local peer - if addr.ID == sp.ID() { - continue - } - // Skip peer already connected - if sp.IsNeighborPeer(addr.ID) { - continue - } - // Skip invalid port - if addr.Port == 0 { - continue - } - // Save to address list - sp.am.AddOrUpdateAddress(&addr) - } - }, - } -} - -func (sp *ServerPeer) Broadcast(msg p2p.Message) { - // Make a copy of neighbor peers list, - // This can prevent mutex lock when peer.Send() - // method fire a disconnect event. - neighbors := sp.GetNeighborPeers() - - // Do broadcast - go func() { - for _, peer := range neighbors { - - // Skip disconnected peer - if !peer.Connected() { - continue - } - - // Skip non relay peer - if peer.Relay() == 0 { - continue - } - - peer.QueueMessage(msg, nil) - } - }() -} diff --git a/sdk/spvpeer.go b/peer/peer.go similarity index 81% rename from sdk/spvpeer.go rename to peer/peer.go index 12f384a..569c1a8 100644 --- a/sdk/spvpeer.go +++ b/peer/peer.go @@ -1,7 +1,6 @@ package sdk import ( - "sync" "time" "github.com/elastos/Elastos.ELA.SPV/log" @@ -17,73 +16,78 @@ import ( const ( // stallTickInterval is the interval of time between each check for // stalled peers. - stallTickInterval = 15 * time.Second + stallTickInterval = 5 * time.Second // stallResponseTimeout is the base maximum amount of time messages that // expect a response will wait before disconnecting the peer for // stalling. The deadlines are adjusted for callback running times and // only checked on each stall tick interval. - stallResponseTimeout = 30 * time.Second + stallResponseTimeout = 15 * time.Second ) -type downloadTx struct { - mutex sync.Mutex - queue map[common.Uint256]struct{} -} +func (p *SPVPeer) EnqueueBlock(header *msg.MerkleBlock, txIds []*common.Uint256) *block { + hash := header.Header.(*core.Header).Hash() + + block := newBlock(header) + + // No transactions to download, just finish it + if len(txIds) == 0 { + return block + } + + // Download transactions of this block + getData := msg.NewGetData() + for _, txId := range txIds { + block.txQueue[*txId] = struct{}{} + getData.AddInvVect(msg.NewInvVect(msg.InvTypeTx, txId)) + } + // Stall message + p.stallControl <- getData -func newDownloadTx() *downloadTx { - return &downloadTx{queue: make(map[common.Uint256]struct{})} + p.blocksQueue[hash] = block + + return nil } -func (d *downloadTx) queueTx(txId common.Uint256) { - d.mutex.Lock() - defer d.mutex.Unlock() - d.queue[txId] = struct{}{} +func (p *SPVPeer) EnqueueTx(txId common.Uint256) { + p.txsQueue[txId] = struct{}{} } -func (d *downloadTx) dequeueTx(txId common.Uint256) bool { - d.mutex.Lock() - defer d.mutex.Unlock() - _, ok := d.queue[txId] +func (p *SPVPeer) DequeueTx(tx *core.Transaction) (download interface{}, finished bool) { + // Get tx id + txId := tx.Hash() + + // Check downloading block first + if blockId, ok := p.blockTxs[txId]; ok { + // Dequeue block tx + block := p.blocksQueue[blockId] + block.txs = append(block.txs, tx) + delete(block.txQueue, txId) + + return block, len(block.txQueue) == 0 + } + + // Dequeue from downloading tx + _, ok := p.txsQueue[txId] if !ok { - return false + return tx, false } - delete(d.queue, txId) - return true + delete(p.txsQueue, txId) + + return tx, true } -type downloadBlock struct { - mutex sync.Mutex +type block struct { *msg.MerkleBlock txQueue map[common.Uint256]struct{} txs []*core.Transaction } -func newDownloadBlock() *downloadBlock { - return &downloadBlock{txQueue: make(map[common.Uint256]struct{})} -} - -func (d *downloadBlock) enqueueTx(txId common.Uint256) { - d.mutex.Lock() - defer d.mutex.Unlock() - d.txQueue[txId] = struct{}{} -} - -func (d *downloadBlock) dequeueTx(txId common.Uint256) bool { - d.mutex.Lock() - defer d.mutex.Unlock() - _, ok := d.txQueue[txId] - if !ok { - return false +func newBlock(header *msg.MerkleBlock) *block { + return &block{ + MerkleBlock: header, + txQueue: make(map[common.Uint256]struct{}), } - delete(d.txQueue, txId) - return true -} - -func (d *downloadBlock) finished() bool { - d.mutex.Lock() - defer d.mutex.Unlock() - return len(d.txQueue) == 0 } type SPVPeerConfig struct { @@ -122,8 +126,9 @@ type SPVPeer struct { *net.Peer blockQueue chan common.Uint256 - downloading *downloadBlock - downloadTx *downloadTx + blocksQueue map[common.Uint256]*block + blockTxs map[common.Uint256]common.Uint256 + txsQueue map[common.Uint256]struct{} receivedTxs int fPositives int @@ -134,12 +139,13 @@ func NewSPVPeer(peer *net.Peer, config SPVPeerConfig) *SPVPeer { spvPeer := &SPVPeer{ Peer: peer, blockQueue: make(chan common.Uint256, p2p.MaxBlocksPerMsg), - downloading: newDownloadBlock(), - downloadTx: newDownloadTx(), + blocksQueue: make(map[common.Uint256]*block), + blockTxs: make(map[common.Uint256]common.Uint256), + txsQueue: make(map[common.Uint256]struct{}), stallControl: make(chan p2p.Message, 1), } - msgConfig := rw.MessageConfig{ + msgConfig := rw.Config{ ProtocolVersion: p2p.EIP001Version, MakeTx: func() *msg.Tx { return msg.NewTx(new(core.Transaction)) }, MakeBlock: func() *msg.Block { return msg.NewBlock(new(core.Block)) }, @@ -267,10 +273,6 @@ cleanup: log.Tracef("Peer stall handler done for %v", p) } -func (p *SPVPeer) StallMessage(message p2p.Message) { - p.stallControl <- message -} - // Add message to output queue and wait until message sent func (p *SPVPeer) SendMessage(message p2p.Message) { doneChan := make(chan struct{}) @@ -291,10 +293,6 @@ func (p *SPVPeer) queueMessage(message p2p.Message, doneChan chan struct{}) { p.Peer.QueueMessage(message, doneChan) } -func (p *SPVPeer) ResetDownloading() { - p.downloading = newDownloadBlock() -} - func (p *SPVPeer) GetFalsePositiveRate() float32 { return float32(p.fPositives) / float32(p.receivedTxs) } diff --git a/sdk/blockchain.go b/sdk/blockchain.go deleted file mode 100644 index e35b232..0000000 --- a/sdk/blockchain.go +++ /dev/null @@ -1,373 +0,0 @@ -package sdk - -import ( - "errors" - "fmt" - "math/big" - "sync" - - "github.com/elastos/Elastos.ELA.SPV/log" - "github.com/elastos/Elastos.ELA.SPV/store" - - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" -) - -type ChainState int - -const ( - SYNCING = ChainState(0) - WAITING = ChainState(1) -) - -func (s ChainState) String() string { - switch s { - case SYNCING: - return "SYNCING" - case WAITING: - return "WAITING" - default: - return "UNKNOWN" - } -} - -const ( - MaxBlockLocatorHashes = 100 -) - -var PowLimit = new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 255), big.NewInt(1)) - -/* -Blockchain is the database of blocks, also when a new transaction or block commit, -Blockchain will verify them with stored blocks. -*/ -type Blockchain struct { - lock *sync.RWMutex - state ChainState - store.HeaderStore -} - -// Create a instance of *Blockchain -func NewBlockchain(foundation string, headerStore store.HeaderStore) (*Blockchain, error) { - blockchain := &Blockchain{ - lock: new(sync.RWMutex), - state: WAITING, - HeaderStore: headerStore, - } - - // Init genesis header - _, err := blockchain.GetBestHeader() - if err != nil { - var err error - var foundationAddress *common.Uint168 - if len(foundation) == 34 { - foundationAddress, err = common.Uint168FromAddress(foundation) - } else { - foundationAddress, err = common.Uint168FromAddress("8VYXVxKKSAxkmRrfmGpQR2Kc66XhG6m3ta") - } - if err != nil { - return nil, errors.New("parse foundation address failed") - } - genesisHeader := GenesisHeader(foundationAddress) - storeHeader := &store.StoreHeader{Header: *genesisHeader, TotalWork: new(big.Int)} - blockchain.PutHeader(storeHeader, true) - } - - return blockchain, nil -} - -// Close the blockchain -func (bc *Blockchain) Close() { - bc.lock.Lock() - bc.HeaderStore.Close() -} - -// Set the current state of blockchain -func (bc *Blockchain) SetChainState(state ChainState) { - bc.lock.Lock() - defer bc.lock.Unlock() - - bc.state = state -} - -// Return a bool value if blockchain is in syncing state -func (bc *Blockchain) IsSyncing() bool { - bc.lock.RLock() - defer bc.lock.RUnlock() - - return bc.state == SYNCING -} - -// Get current blockchain height -func (bc *Blockchain) Height() uint32 { - bc.lock.RLock() - defer bc.lock.RUnlock() - - tip, err := bc.GetBestHeader() - if err != nil { - return 0 - } - return tip.Height -} - -// Get current blockchain tip -func (bc *Blockchain) ChainTip() *store.StoreHeader { - bc.lock.RLock() - defer bc.lock.RUnlock() - - return bc.chainTip() -} - -func (bc *Blockchain) chainTip() *store.StoreHeader { - tip, err := bc.GetBestHeader() - if err != nil { // Empty blockchain, return empty header - return &store.StoreHeader{TotalWork: new(big.Int)} - } - return tip -} - -// Create a block locator which is a array of block hashes stored in blockchain -func (bc *Blockchain) GetBlockLocatorHashes() []*common.Uint256 { - bc.lock.RLock() - defer bc.lock.RUnlock() - - var ret []*common.Uint256 - parent, err := bc.GetBestHeader() - if err != nil { // No headers stored return empty locator - return ret - } - - rollback := func(parent *store.StoreHeader, n int) (*store.StoreHeader, error) { - for i := 0; i < n; i++ { - parent, err = bc.GetPrevious(parent) - if err != nil { - return parent, err - } - } - return parent, nil - } - - step := 1 - start := 0 - for { - if start >= 9 { - step *= 2 - start = 0 - } - hash := parent.Hash() - ret = append(ret, &hash) - if len(ret) >= MaxBlockLocatorHashes { - break - } - parent, err = rollback(parent, step) - if err != nil { - break - } - start += 1 - } - return ret -} - -// IsKnownHeader returns if a header is already stored in database by it's hash -func (bc *Blockchain) IsKnownHeader(hash *common.Uint256) bool { - bc.lock.Lock() - defer bc.lock.Unlock() - header, _ := bc.HeaderStore.GetHeader(hash) - return header != nil -} - -// Commit header add a header into blockchain, return if this header -// is a new tip, or meet a reorganize (reorgFrom > 0), and error -func (bc *Blockchain) CommitHeader(header core.Header) (newTip bool, reorgFrom uint32, err error) { - bc.lock.Lock() - defer bc.lock.Unlock() - - err = bc.CheckProofOfWork(header) - if err != nil { - return newTip, reorgFrom, err - } - - commitHeader := &store.StoreHeader{Header: header} - - // Get current chain tip - tip := bc.chainTip() - tipHash := tip.Hash() - - // Lookup of the parent header. Otherwise (ophan?) we need to fetch the parent. - // If the tip is also the parent of this header, then we can save a database read by skipping - var parentHeader *store.StoreHeader - if header.Previous.IsEqual(tipHash) { - parentHeader = tip - } else { - parentHeader, err = bc.GetPrevious(commitHeader) - if err != nil { - return newTip, reorgFrom, fmt.Errorf("Header %s does not extend any known headers", header.Hash().String()) - } - } - - // If this block is already the tip, return - if tipHash.IsEqual(header.Hash()) { - return newTip, reorgFrom, err - } - // Add the work of this header to the total work stored at the previous header - cumulativeWork := new(big.Int).Add(parentHeader.TotalWork, CalcWork(header.Bits)) - commitHeader.TotalWork = cumulativeWork - - // If the cumulative work is greater than the total work of our best header - // then we have a new best header. Update the chain tip and check for a reorg. - var forkPoint *store.StoreHeader - if cumulativeWork.Cmp(tip.TotalWork) > 0 { - newTip = true - // If this header is not extending the previous best header then we have a reorg. - if !tipHash.IsEqual(parentHeader.Hash()) { - commitHeader.Height = parentHeader.Height + 1 - forkPoint, err = bc.getCommonAncestor(commitHeader, tip) - if err != nil { - log.Errorf("error calculating common ancestor: %s", err.Error()) - return newTip, reorgFrom, err - } - log.Infof("Reorganize At block %d, Wiped out %d blocks", - int(tip.Height), int(tip.Height-forkPoint.Height)) - } - } - - // If common ancestor exists, means we have an fork chan - // so we need to rollback to the last good point. - if bc.state != SYNCING && forkPoint != nil { - reorgFrom = tip.Height - // Save reorganize point as the new tip - err = bc.PutHeader(forkPoint, newTip) - if err != nil { - return newTip, reorgFrom, err - } - return newTip, reorgFrom, err - } - - // Save header to db - err = bc.PutHeader(commitHeader, newTip) - if err != nil { - return newTip, reorgFrom, err - } - - return newTip, reorgFrom, err -} - -// Returns last header before reorg point -func (bc *Blockchain) getCommonAncestor(bestHeader, prevTip *store.StoreHeader) (*store.StoreHeader, error) { - var err error - rollback := func(parent *store.StoreHeader, n int) (*store.StoreHeader, error) { - for i := 0; i < n; i++ { - parent, err = bc.GetPrevious(parent) - if err != nil { - return parent, err - } - } - return parent, nil - } - - majority := bestHeader - minority := prevTip - if bestHeader.Height > prevTip.Height { - majority, err = rollback(majority, int(bestHeader.Height-prevTip.Height)) - if err != nil { - return nil, err - } - } else if prevTip.Height > bestHeader.Height { - minority, err = rollback(minority, int(prevTip.Height-bestHeader.Height)) - if err != nil { - return nil, err - } - } - - for { - majorityHash := majority.Hash() - minorityHash := minority.Hash() - if majorityHash.IsEqual(minorityHash) { - return majority, nil - } - majority, err = bc.GetPrevious(majority) - if err != nil { - return nil, err - } - minority, err = bc.GetPrevious(minority) - if err != nil { - return nil, err - } - } -} - -func CalcWork(bits uint32) *big.Int { - // Return a work value of zero if the passed difficulty bits represent - // a negative number. Note this should not happen in practice with valid - // blocks, but an invalid block could trigger it. - difficultyNum := CompactToBig(bits) - if difficultyNum.Sign() <= 0 { - return big.NewInt(0) - } - - // (1 << 256) / (difficultyNum + 1) - denominator := new(big.Int).Add(difficultyNum, big.NewInt(1)) - return new(big.Int).Div(new(big.Int).Lsh(big.NewInt(1), 256), denominator) -} - -func (bc *Blockchain) CheckProofOfWork(header core.Header) error { - // The target difficulty must be larger than zero. - target := CompactToBig(header.Bits) - if target.Sign() <= 0 { - return errors.New("[Blockchain], block target difficulty is too low.") - } - - // The target difficulty must be less than the maximum allowed. - if target.Cmp(PowLimit) > 0 { - return errors.New("[Blockchain], block target difficulty is higher than max of limit.") - } - - // The block hash must be less than the claimed target. - hash := header.AuxPow.ParBlockHeader.Hash() - hashNum := HashToBig(&hash) - if hashNum.Cmp(target) > 0 { - return errors.New("[Blockchain], block target difficulty is higher than expected difficulty.") - } - - return nil -} - -func HashToBig(hash *common.Uint256) *big.Int { - // A Hash is in little-endian, but the big package wants the bytes in - // big-endian, so reverse them. - buf := *hash - blen := len(buf) - for i := 0; i < blen/2; i++ { - buf[i], buf[blen-1-i] = buf[blen-1-i], buf[i] - } - - return new(big.Int).SetBytes(buf[:]) -} - -func CompactToBig(compact uint32) *big.Int { - // Extract the mantissa, sign bit, and exponent. - mantissa := compact & 0x007fffff - isNegative := compact&0x00800000 != 0 - exponent := uint(compact >> 24) - - // Since the base for the exponent is 256, the exponent can be treated - // as the number of bytes to represent the full 256-bit number. So, - // treat the exponent as the number of bytes and shift the mantissa - // right or left accordingly. This is equivalent to: - // N = mantissa * 256^(exponent-3) - var bn *big.Int - if exponent <= 3 { - mantissa >>= 8 * (3 - exponent) - bn = big.NewInt(int64(mantissa)) - } else { - bn = big.NewInt(int64(mantissa)) - bn.Lsh(bn, 8*(exponent-3)) - } - - // Make it negative if the sign bit is set. - if isNegative { - bn = bn.Neg(bn) - } - - return bn -} diff --git a/sdk/interface.go b/sdk/interface.go new file mode 100644 index 0000000..3ae667e --- /dev/null +++ b/sdk/interface.go @@ -0,0 +1,81 @@ +package sdk + +import ( + "github.com/elastos/Elastos.ELA.SPV/store" + "github.com/elastos/Elastos.ELA.Utility/p2p/server" + + "github.com/elastos/Elastos.ELA.SPV/net" + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + ela "github.com/elastos/Elastos.ELA/core" +) + +/* +SPV service is a high level implementation with all SPV logic implemented. +SPV service is extend from SPV client and implement BlockChain and block synchronize on it. +With SPV service, you just need to implement your own HeaderStore and SPVServiceConfig, and let other stuff go. +*/ +type SPVService interface { + // Start SPV service + Start() + + // Stop SPV service + Stop() + + // IsSyncing returns the current state of the service, to indicate that the service + // is in syncing mode or waiting mode. + IsSyncing() bool + + // SendTransaction broadcast a transaction message to the peer to peer network. + SendTransaction(ela.Transaction) (*common.Uint256, error) +} + +type SPVServiceConfig struct { + // The server access into blockchain peer to peer network + Server server.IServer + + // Foundation address of the current access blockhain network + Foundation string + + // The database to store all block headers + HeaderStore store.HeaderStore + + // GetFilterData() returns two arguments. + // First arguments are all addresses stored in your data store. + // Second arguments are all balance references to those addresses stored in your data store, + // including UTXO(Unspent Transaction Output)s and STXO(Spent Transaction Output)s. + // Outpoint is a data structure include a transaction ID and output index. It indicates the + // reference of an transaction output. If an address ever received an transaction output, + // there will be the outpoint reference to it. Any time you want to spend the balance of an + // address, you must provide the reference of the balance which is an outpoint in the transaction input. + GetFilterData func() ([]*common.Uint168, []*ela.OutPoint) + + // When interested transactions received, this method will call back them. + // The height is the block height where this transaction has been packed. + // Returns if the transaction is a match, for there will be transactions that + // are not interested go through this method. If a transaction is not a match + // return false as a false positive mark. If anything goes wrong, return error. + // Notice: this method will be callback when commit block + CommitTx func(tx *ela.Transaction, height uint32) (bool, error) + + // This method will be callback after a block and transactions with it are + // successfully committed into database. + OnBlockCommitted func(*msg.MerkleBlock, []*ela.Transaction) + + // When the blockchain meet a reorganization, data should be rollback to the fork point. + // The Rollback method will callback the current rollback height, for example OnChainRollback(100) + // means data on height 100 has been deleted, current chain height will be 99. You should rollback + // stored data including UTXOs STXOs Txs etc. according to the given height. + // If anything goes wrong, return an error. + OnRollback func(height uint32) error +} + +/* +Get a SPV service instance. +there are two implementations you need to do, DataStore and GetBloomFilter() method. +DataStore is an interface including all methods you need to implement placed in db/datastore.go. +Also an sample APP spvwallet is contain in this project placed in spvwallet folder. +*/ +func GetSPVService(config SPVServiceConfig) (SPVService, error) { + return NewSPVServiceImpl(config) +} diff --git a/sdk/spvservice.go b/sdk/spvservice.go index 1fc6c7c..96c0dee 100644 --- a/sdk/spvservice.go +++ b/sdk/spvservice.go @@ -1,85 +1,335 @@ package sdk import ( - "github.com/elastos/Elastos.ELA.SPV/store" + "errors" + "fmt" + "time" + "github.com/elastos/Elastos.ELA.SPV/log" "github.com/elastos/Elastos.ELA.SPV/net" + "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - ela "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/bloom" + "github.com/elastos/Elastos.ELA/core" +) + +const ( + SendTxTimeout = time.Second * 10 ) -/* -SPV service is a high level implementation with all SPV logic implemented. -SPV service is extend from SPV client and implement Blockchain and block synchronize on it. -With SPV service, you just need to implement your own HeaderStore and SPVServiceConfig, and let other stuff go. -*/ -type SPVService interface { - // Start SPV service - Start() - - // Stop SPV service - Stop() - - // ChainState returns the current state of the blockchain, to indicate that the blockchain - // is in syncing mode or waiting mode. - ChainState() ChainState - - // ReloadFilters is a trigger to make SPV service refresh the current - // transaction filer(in our implementation the bloom filter) in SPV service. - // This will call onto the GetAddresses() and GetOutpoints() method in SPVServiceConfig. - ReloadFilter() - - // SendTransaction broadcast a transaction message to the peer to peer network. - SendTransaction(ela.Transaction) (*common.Uint256, error) -} - -type SPVServiceConfig struct { - // The server peer access into blockchain peer to peer network - Server *net.ServerPeer - - // Foundation address of the current access blockhain network - Foundation string - - // The database to store all block headers - HeaderStore store.HeaderStore - - // GetFilterData() returns two arguments. - // First arguments are all addresses stored in your data store. - // Second arguments are all balance references to those addresses stored in your data store, - // including UTXO(Unspent Transaction Output)s and STXO(Spent Transaction Output)s. - // Outpoint is a data structure include a transaction ID and output index. It indicates the - // reference of an transaction output. If an address ever received an transaction output, - // there will be the outpoint reference to it. Any time you want to spend the balance of an - // address, you must provide the reference of the balance which is an outpoint in the transaction input. - GetFilterData func() ([]*common.Uint168, []*ela.OutPoint) - - // When interested transactions received, this method will call back them. - // The height is the block height where this transaction has been packed. - // Returns if the transaction is a match, for there will be transactions that - // are not interested go through this method. If a transaction is not a match - // return false as a false positive mark. If anything goes wrong, return error. - // Notice: this method will be callback when commit block - CommitTx func(tx *ela.Transaction, height uint32) (bool, error) - - // This method will be callback after a block and transactions with it are - // successfully committed into database. - OnBlockCommitted func(*msg.MerkleBlock, []*ela.Transaction) - - // When the blockchain meet a reorganization, data should be rollback to the fork point. - // The Rollback method will callback the current rollback height, for example OnChainRollback(100) - // means data on height 100 has been deleted, current chain height will be 99. You should rollback - // stored data including UTXOs STXOs Txs etc. according to the given height. - // If anything goes wrong, return an error. - OnRollback func(height uint32) error -} - -/* -Get a SPV service instance. -there are two implementations you need to do, DataStore and GetBloomFilter() method. -DataStore is an interface including all methods you need to implement placed in db/datastore.go. -Also an sample APP spvwallet is contain in this project placed in spvwallet folder. -*/ -func GetSPVService(config SPVServiceConfig) (SPVService, error) { - return NewSPVServiceImpl(config) +// The SPV service implementation +type SPVServiceImpl struct { + *net.ServerPeer + syncManager *SyncManager + chain *BlockChain + pendingTx common.Uint256 + txAccept chan *common.Uint256 + txReject chan *msg.Reject + config SPVServiceConfig +} + +// Create a instance of SPV service implementation. +func NewSPVServiceImpl(config SPVServiceConfig) (*SPVServiceImpl, error) { + // Initialize blockchain + chain, err := NewBlockchain(config.Foundation, config.HeaderStore) + if err != nil { + return nil, err + } + + // Create SPV service instance + service := &SPVServiceImpl{ + ServerPeer: config.Server, + chain: chain, + config: config, + } + + // Create sync manager config + syncConfig := SyncManageConfig{ + LocalHeight: chain.BestHeight, + GetBlocks: service.GetBlocks, + } + + service.syncManager = NewSyncManager(syncConfig) + + // Set manage config + service.SetConfig(net.PeerManageConfig{ + OnHandshake: service.OnHandshake, + OnPeerEstablish: service.OnPeerEstablish, + }) + + return service, nil +} + +func (s *SPVServiceImpl) OnHandshake(v *msg.Version) error { + if v.Services/OpenService&1 == 0 { + return errors.New("SPV service not enabled on connected peer") + } + + return nil +} + +func (s *SPVServiceImpl) OnPeerEstablish(peer *net.Peer) { + // Create spv peer config + config := SPVPeerConfig{ + LocalHeight: s.LocalHeight, + OnInventory: s.OnInventory, + OnMerkleBlock: s.OnMerkleBlock, + OnTx: s.OnTx, + OnNotFound: s.OnNotFound, + OnReject: s.OnReject, + } + + s.syncManager.AddNeighborPeer(NewSPVPeer(peer, config)) + + // Load bloom filter + doneChan := make(chan struct{}) + peer.QueueMessage(s.BloomFilter(), doneChan) + <-doneChan +} + +func (s *SPVServiceImpl) LocalHeight() uint32 { + return uint32(s.ServerPeer.Height()) +} + +func (s *SPVServiceImpl) Start() { + s.ServerPeer.Start() + s.syncManager.start() + log.Info("SPV service started...") +} + +func (s *SPVServiceImpl) Stop() { + s.chain.Close() + log.Info("SPV service stopped...") +} + +func (s *SPVServiceImpl) ChainState() ChainState { + return s.chain.state +} + +func (s *SPVServiceImpl) ReloadFilter() { + log.Debug() + s.Broadcast(BuildBloomFilter(s.config.GetFilterData()).GetFilterLoadMsg()) +} + +func (s *SPVServiceImpl) SendTransaction(tx core.Transaction) (*common.Uint256, error) { + log.Debug() + + if s.GetNeighborCount() == 0 { + return nil, fmt.Errorf("method not available, no peers connected") + } + + s.txAccept = make(chan *common.Uint256, 1) + s.txReject = make(chan *msg.Reject, 1) + + finish := func() { + close(s.txAccept) + close(s.txReject) + s.txAccept = nil + s.txReject = nil + } + // Set transaction in pending + s.pendingTx = tx.Hash() + // Broadcast transaction to neighbor peers + s.Broadcast(msg.NewTx(&tx)) + // Query neighbors mempool see if transaction was successfully added to mempool + s.Broadcast(new(msg.MemPool)) + + // Wait for result + timer := time.NewTimer(SendTxTimeout) + select { + case <-timer.C: + finish() + return nil, fmt.Errorf("Send transaction timeout") + case <-s.txAccept: + timer.Stop() + finish() + // commit unconfirmed transaction to db + _, err := s.config.CommitTx(&tx, 0) + return &s.pendingTx, err + case msg := <-s.txReject: + timer.Stop() + finish() + return nil, fmt.Errorf("Transaction rejected Code: %s, Reason: %s", msg.Code.String(), msg.Reason) + } +} + +func (s *SPVServiceImpl) GetBlocks() *msg.GetBlocks { + // Get blocks returns a inventory message which contains block hashes + locator := s.chain.GetBlockLocatorHashes() + return msg.NewGetBlocks(locator, common.EmptyHash) +} + +func (s *SPVServiceImpl) BloomFilter() *msg.FilterLoad { + bloomFilter := BuildBloomFilter(s.config.GetFilterData()) + return bloomFilter.GetFilterLoadMsg() +} + +func (s *SPVServiceImpl) OnInventory(peer *SPVPeer, m *msg.Inventory) error { + getData := msg.NewGetData() + + for _, inv := range m.InvList { + switch inv.Type { + case msg.InvTypeBlock: + // Filter duplicated block + if s.chain.IsKnownHeader(&inv.Hash) { + continue + } + + // Kind of lame to send separate getData messages but this allows us + // to take advantage of the timeout on the upper layer. Otherwise we + // need separate timeout handling. + inv.Type = msg.InvTypeFilteredBlock + getData.AddInvVect(inv) + if s.syncManager.IsSyncPeer(peer) { + peer.blockQueue <- inv.Hash + } + + case msg.InvTypeTx: + if s.txAccept != nil && s.pendingTx.IsEqual(inv.Hash) { + s.txAccept <- nil + continue + } + getData.AddInvVect(inv) + peer.EnqueueTx(inv.Hash) + + default: + continue + } + } + + if len(getData.InvList) > 0 { + peer.QueueMessage(getData) + } + return nil +} + +func (s *SPVServiceImpl) OnMerkleBlock(peer *SPVPeer, mBlock *msg.MerkleBlock) error { + blockHash := mBlock.Header.(*core.Header).Hash() + + // Merkleblock from sync peer + if s.syncManager.IsSyncPeer(peer) { + queueHash := <-peer.blockQueue + if !blockHash.IsEqual(queueHash) { + peer.Disconnect() + return fmt.Errorf("peer %d is sending us blocks out of order", peer.ID()) + } + } + + txIds, err := bloom.CheckMerkleBlock(*mBlock) + if err != nil { + return fmt.Errorf("invalid merkleblock received %s", err.Error()) + } + + dBlock := peer.EnqueueBlock(mBlock, txIds) + if dBlock != nil { + s.commitBlock(peer, dBlock) + + // Try continue sync progress + s.syncManager.ContinueSync() + + } + + return nil +} + +func (s *SPVServiceImpl) OnTx(peer *SPVPeer, msg *msg.Tx) error { + tx := msg.Transaction.(*core.Transaction) + + obj, ok := peer.DequeueTx(tx) + if ok { + switch obj := obj.(type) { + case *block: + // commit block + s.commitBlock(peer, obj) + + // Try continue sync progress + s.syncManager.ContinueSync() + + case *core.Transaction: + // commit unconfirmed transaction + _, err := s.config.CommitTx(tx, 0) + if err == nil { + // Update bloom filter + peer.SendMessage(s.BloomFilter()) + } + + return err + } + } + + return fmt.Errorf("Transaction not found in download queue %s", tx.Hash().String()) +} + +func (s *SPVServiceImpl) OnNotFound(peer *SPVPeer, notFound *msg.NotFound) error { + for _, iv := range notFound.InvList { + log.Warnf("Data not found type %s, hash %s", iv.Type.String(), iv.Hash.String()) + } + return nil +} + +func (s *SPVServiceImpl) OnReject(peer *SPVPeer, msg *msg.Reject) error { + if s.pendingTx.IsEqual(msg.Hash); s.txReject != nil { + s.txReject <- msg + return nil + } + return fmt.Errorf("Received reject message from peer %d: Code: %s, Hash %s, Reason: %s", + peer.ID(), msg.Code.String(), msg.Hash.String(), msg.Reason) +} + +func (s *SPVServiceImpl) commitBlock(peer *SPVPeer, block *block) { + header := block.Header.(*core.Header) + newTip, reorgFrom, err := s.chain.CommitHeader(*header) + if err != nil { + log.Errorf("Commit header failed %s", err.Error()) + return + } + if !newTip { + return + } + + newHeight := s.chain.BestHeight() + if reorgFrom > 0 { + for i := reorgFrom; i > newHeight; i-- { + if err = s.config.OnRollback(i); err != nil { + log.Errorf("Rollback transaction at height %d failed %s", i, err.Error()) + return + } + } + + if !s.chain.IsSyncing() { + s.syncManager.StartSync() + return + } + } + + for _, tx := range block.txs { + // Increase received transaction count + peer.receivedTxs++ + + falsePositive, err := s.config.CommitTx(tx, header.Height) + if err != nil { + log.Errorf("Commit transaction %s failed %s", tx.Hash().String(), err.Error()) + return + } + + // Increase false positive count + if falsePositive { + peer.fPositives++ + } + } + + // Refresh bloom filter if false positives meet target rate + if peer.GetFalsePositiveRate() > FalsePositiveRate { + // Reset false positives + peer.ResetFalsePositives() + + // Update bloom filter + peer.SendMessage(s.BloomFilter()) + } + + s.ServerPeer.SetHeight(uint64(newHeight)) + + // Notify block committed + go s.config.OnBlockCommitted(block.MerkleBlock, block.txs) } diff --git a/sdk/spvserviceimpl.go b/sdk/spvserviceimpl.go deleted file mode 100644 index e2acc47..0000000 --- a/sdk/spvserviceimpl.go +++ /dev/null @@ -1,367 +0,0 @@ -package sdk - -import ( - "errors" - "fmt" - "time" - - "github.com/elastos/Elastos.ELA.SPV/log" - "github.com/elastos/Elastos.ELA.SPV/net" - - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA/bloom" - "github.com/elastos/Elastos.ELA/core" -) - -const ( - SendTxTimeout = time.Second * 10 -) - -// The SPV service implementation -type SPVServiceImpl struct { - *net.ServerPeer - syncManager *SyncManager - chain *Blockchain - pendingTx common.Uint256 - txAccept chan *common.Uint256 - txReject chan *msg.Reject - config SPVServiceConfig -} - -// Create a instance of SPV service implementation. -func NewSPVServiceImpl(config SPVServiceConfig) (*SPVServiceImpl, error) { - // Initialize blockchain - chain, err := NewBlockchain(config.Foundation, config.HeaderStore) - if err != nil { - return nil, err - } - - // Create SPV service instance - service := &SPVServiceImpl{ - ServerPeer: config.Server, - chain: chain, - config: config, - } - - // Create sync manager config - syncConfig := SyncManageConfig{ - LocalHeight: chain.Height, - GetBlocks: service.GetBlocks, - } - - service.syncManager = NewSyncManager(syncConfig) - - // Set manage config - service.SetConfig(net.PeerManageConfig{ - OnHandshake: service.OnHandshake, - OnPeerEstablish: service.OnPeerEstablish, - }) - - return service, nil -} - -func (s *SPVServiceImpl) OnHandshake(v *msg.Version) error { - if v.Services/OpenService&1 == 0 { - return errors.New("SPV service not enabled on connected peer") - } - - return nil -} - -func (s *SPVServiceImpl) OnPeerEstablish(peer *net.Peer) { - // Create spv peer config - config := SPVPeerConfig{ - LocalHeight: s.LocalHeight, - OnInventory: s.OnInventory, - OnMerkleBlock: s.OnMerkleBlock, - OnTx: s.OnTx, - OnNotFound: s.OnNotFound, - OnReject: s.OnReject, - } - - s.syncManager.AddNeighborPeer(NewSPVPeer(peer, config)) - - // Load bloom filter - doneChan := make(chan struct{}) - peer.QueueMessage(s.BloomFilter(), doneChan) - <-doneChan -} - -func (s *SPVServiceImpl) LocalHeight() uint32 { - return uint32(s.ServerPeer.Height()) -} - -func (s *SPVServiceImpl) Start() { - s.ServerPeer.Start() - s.syncManager.start() - log.Info("SPV service started...") -} - -func (s *SPVServiceImpl) Stop() { - s.chain.Close() - log.Info("SPV service stopped...") -} - -func (s *SPVServiceImpl) ChainState() ChainState { - return s.chain.state -} - -func (s *SPVServiceImpl) ReloadFilter() { - log.Debug() - s.Broadcast(BuildBloomFilter(s.config.GetFilterData()).GetFilterLoadMsg()) -} - -func (s *SPVServiceImpl) SendTransaction(tx core.Transaction) (*common.Uint256, error) { - log.Debug() - - if s.GetNeighborCount() == 0 { - return nil, fmt.Errorf("method not available, no peers connected") - } - - s.txAccept = make(chan *common.Uint256, 1) - s.txReject = make(chan *msg.Reject, 1) - - finish := func() { - close(s.txAccept) - close(s.txReject) - s.txAccept = nil - s.txReject = nil - } - // Set transaction in pending - s.pendingTx = tx.Hash() - // Broadcast transaction to neighbor peers - s.Broadcast(msg.NewTx(&tx)) - // Query neighbors mempool see if transaction was successfully added to mempool - s.Broadcast(new(msg.MemPool)) - - // Wait for result - timer := time.NewTimer(SendTxTimeout) - select { - case <-timer.C: - finish() - return nil, fmt.Errorf("Send transaction timeout") - case <-s.txAccept: - timer.Stop() - finish() - // commit unconfirmed transaction to db - _, err := s.config.CommitTx(&tx, 0) - return &s.pendingTx, err - case msg := <-s.txReject: - timer.Stop() - finish() - return nil, fmt.Errorf("Transaction rejected Code: %s, Reason: %s", msg.Code.String(), msg.Reason) - } -} - -func (s *SPVServiceImpl) GetBlocks() *msg.GetBlocks { - // Get blocks returns a inventory message which contains block hashes - locator := s.chain.GetBlockLocatorHashes() - return msg.NewGetBlocks(locator, common.EmptyHash) -} - -func (s *SPVServiceImpl) BloomFilter() *msg.FilterLoad { - bloomFilter := BuildBloomFilter(s.config.GetFilterData()) - return bloomFilter.GetFilterLoadMsg() -} - -func (s *SPVServiceImpl) OnInventory(peer *SPVPeer, m *msg.Inventory) error { - getData := msg.NewGetData() - - for _, inv := range m.InvList { - switch inv.Type { - case msg.InvTypeBlock: - // Filter duplicated block - if s.chain.IsKnownHeader(&inv.Hash) { - continue - } - - // Kind of lame to send separate getData messages but this allows us - // to take advantage of the timeout on the upper layer. Otherwise we - // need separate timeout handling. - inv.Type = msg.InvTypeFilteredBlock - getData.AddInvVect(inv) - if s.syncManager.IsSyncPeer(peer) { - peer.blockQueue <- inv.Hash - } - - case msg.InvTypeTx: - if s.txAccept != nil && s.pendingTx.IsEqual(inv.Hash) { - s.txAccept <- nil - continue - } - getData.AddInvVect(inv) - peer.downloadTx.queueTx(inv.Hash) - - default: - continue - } - } - - if len(getData.InvList) > 0 { - peer.QueueMessage(getData) - } - return nil -} - -func (s *SPVServiceImpl) OnMerkleBlock(peer *SPVPeer, block *msg.MerkleBlock) error { - blockHash := block.Header.(*core.Header).Hash() - - // Merkleblock from sync peer - if s.syncManager.IsSyncPeer(peer) { - queueHash := <-peer.blockQueue - if !blockHash.IsEqual(queueHash) { - peer.Disconnect() - return fmt.Errorf("peer %d is sending us blocks out of order", peer.ID()) - } - } - - txIds, err := bloom.CheckMerkleBlock(*block) - if err != nil { - return fmt.Errorf("invalid merkleblock received %s", err.Error()) - } - - // Save block as download block - peer.downloading.MerkleBlock = block - - // No transactions to download, just finish it - if len(txIds) == 0 { - s.finishDownloading(peer) - return nil - } - - // Download transactions of this block - getData := msg.NewGetData() - for _, txId := range txIds { - getData.AddInvVect(msg.NewInvVect(msg.InvTypeTx, txId)) - peer.downloading.enqueueTx(*txId) - } - // Stall message - peer.StallMessage(getData) - - return nil -} - -func (s *SPVServiceImpl) OnTx(peer *SPVPeer, msg *msg.Tx) error { - tx := msg.Transaction.(*core.Transaction) - if peer.downloadTx.dequeueTx(tx.Hash()) { - // commit unconfirmed transaction - _, err := s.config.CommitTx(tx, 0) - if err == nil { - // Update bloom filter - peer.SendMessage(s.BloomFilter()) - } - return err - } - - if !peer.downloading.dequeueTx(tx.Hash()) { - peer.downloading = newDownloadBlock() - return fmt.Errorf("Transaction not found in download queue %s", tx.Hash().String()) - } - - // Add tx to download - peer.downloading.txs = append(peer.downloading.txs, tx) - - // All transactions of the download block have been received, commit the download block - if peer.downloading.finished() { - // Finish current downloading block - s.finishDownloading(peer) - - } - - return nil -} - -func (s *SPVServiceImpl) OnNotFound(peer *SPVPeer, notFound *msg.NotFound) error { - for _, iv := range notFound.InvList { - log.Warnf("Data not found type %s, hash %s", iv.Type.String(), iv.Hash.String()) - switch iv.Type { - case msg.InvTypeTx: - if peer.downloadTx.dequeueTx(iv.Hash) { - } - - if peer.downloading.dequeueTx(iv.Hash) { - peer.ResetDownloading() - return nil - } - - case msg.InvTypeBlock: - peer.ResetDownloading() - } - } - return nil -} - -func (s *SPVServiceImpl) OnReject(peer *SPVPeer, msg *msg.Reject) error { - if s.pendingTx.IsEqual(msg.Hash); s.txReject != nil { - s.txReject <- msg - return nil - } - return fmt.Errorf("Received reject message from peer %d: Code: %s, Hash %s, Reason: %s", - peer.ID(), msg.Code.String(), msg.Hash.String(), msg.Reason) -} - -func (s *SPVServiceImpl) finishDownloading(peer *SPVPeer) { - // Commit downloaded block - s.commitBlock(peer) - - peer.ResetDownloading() - - s.syncManager.ContinueSync() -} - -func (s *SPVServiceImpl) commitBlock(peer *SPVPeer) { - block := peer.downloading - header := block.Header.(*core.Header) - newTip, reorgFrom, err := s.chain.CommitHeader(*header) - if err != nil { - log.Errorf("Commit header failed %s", err.Error()) - return - } - if !newTip { - return - } - - newHeight := s.chain.Height() - if reorgFrom > 0 { - for i := reorgFrom; i > newHeight; i-- { - if err = s.config.OnRollback(i); err != nil { - log.Errorf("Rollback transaction at height %d failed %s", i, err.Error()) - return - } - } - - if !s.chain.IsSyncing() { - s.syncManager.StartSyncing() - return - } - } - - for _, tx := range block.txs { - // Increase received transaction count - peer.receivedTxs++ - - falsePositive, err := s.config.CommitTx(tx, header.Height) - if err != nil { - log.Errorf("Commit transaction %s failed %s", tx.Hash().String(), err.Error()) - return - } - - // Increase false positive count - if falsePositive { - peer.fPositives++ - } - } - - // Refresh bloom filter if false positives meet target rate - if peer.GetFalsePositiveRate() > FalsePositiveRate { - // Reset false positives - peer.ResetFalsePositives() - - // Update bloom filter - peer.SendMessage(s.BloomFilter()) - } - - s.ServerPeer.SetHeight(uint64(newHeight)) - s.config.OnBlockCommitted(block.MerkleBlock, block.txs) -} diff --git a/sdk/syncmanager.go b/sdk/syncmanager.go deleted file mode 100644 index d2f56a8..0000000 --- a/sdk/syncmanager.go +++ /dev/null @@ -1,184 +0,0 @@ -package sdk - -import ( - "sync" - "time" - - "github.com/elastos/Elastos.ELA.SPV/log" - - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" -) - -const ( - SyncTickInterval = time.Second * 5 - FalsePositiveRate = float32(1) / float32(1000) -) - -type neighbors struct { - sync.Mutex - list map[uint64]*SPVPeer -} - -func (ns *neighbors) init() { - ns.list = make(map[uint64]*SPVPeer) -} - -func (ns *neighbors) addNeighbor(peer *SPVPeer) { - // Add peer to list - ns.Lock() - ns.list[peer.ID()] = peer - ns.Unlock() -} - -func (ns *neighbors) delNeighbor(id uint64) { - ns.Lock() - delete(ns.list, id) - ns.Unlock() -} - -func (ns *neighbors) getNeighborPeers() []*SPVPeer { - ns.Lock() - defer ns.Unlock() - - peers := make([]*SPVPeer, 0, len(ns.list)) - for _, peer := range ns.list { - if !peer.Connected() { - continue - } - - peers = append(peers, peer) - } - - return peers -} - -func (ns *neighbors) getBestPeer() *SPVPeer { - ns.Lock() - defer ns.Unlock() - var best *SPVPeer - for _, peer := range ns.list { - // Skip disconnected peer - if !peer.Connected() { - continue - } - - // Init best peer - if best == nil { - best = peer - continue - } - - if peer.Height() > best.Height() { - best = peer - } - } - - return best -} - -type SyncManageConfig struct { - LocalHeight func() uint32 - GetBlocks func() *msg.GetBlocks -} - -type SyncManager struct { - config SyncManageConfig - syncPeer *SPVPeer - neighbors -} - -func NewSyncManager(config SyncManageConfig) *SyncManager { - return &SyncManager{config: config} -} - -func (s *SyncManager) start() { - // Initial neighbor list - s.neighbors.init() - - // Start sync handler - go s.syncHandler() -} - -func (s *SyncManager) syncHandler() { - // Check if need sync by SyncTickInterval - ticker := time.NewTicker(SyncTickInterval) - defer ticker.Stop() - - for range ticker.C { - // Try to start a syncing progress - s.StartSyncing() - } -} - -func (s *SyncManager) needSync() (*SPVPeer, bool) { - // Printout neighbor peers height - peers := s.getNeighborPeers() - heights := make([]uint64, 0, len(peers)) - for _, peer := range peers { - heights = append(heights, peer.Height()) - } - log.Info("Neighbors -->", heights, s.config.LocalHeight()) - - bestPeer := s.getBestPeer() - if bestPeer == nil { // no peers connected, return false - log.Info("no peers connected") - return nil, false - } - return bestPeer, bestPeer.Height() > uint64(s.config.LocalHeight()) -} - -func (s *SyncManager) GetBlocks() { - s.syncPeer.QueueMessage(s.config.GetBlocks()) -} - -func (s *SyncManager) AddNeighborPeer(peer *SPVPeer) { - // Wait for peer quit - go func() { - select { - case <-peer.QuitChan(): - if s.syncPeer != nil && s.syncPeer.ID() == peer.ID() { - s.syncPeer = nil - } - s.delNeighbor(peer.ID()) - } - }() - - // Set handler's peer - s.addNeighbor(peer) -} - -func (s *SyncManager) StartSyncing() { - // Check if blockchain need sync - if bestPeer, needSync := s.needSync(); needSync { - // Return if already in syncing - if s.syncPeer != nil { - return - } - - // Set sync peer - s.syncPeer = bestPeer - - // Send getblocks to sync peer - s.GetBlocks() - - } else { - // Return if not in syncing - if s.syncPeer == nil { - return - } - - // Clear sync peer - s.syncPeer = nil - - } -} - -func (s *SyncManager) ContinueSync() { - if s.syncPeer != nil && len(s.syncPeer.blockQueue) == 0 { - s.GetBlocks() - } -} - -func (s *SyncManager) IsSyncPeer(peer *SPVPeer) bool { - return s.syncPeer != nil && s.syncPeer.ID() == peer.ID() -} diff --git a/spvwallet/cli/account/account.go b/spvwallet/client/account/account.go similarity index 99% rename from spvwallet/cli/account/account.go rename to spvwallet/client/account/account.go index 7cecd73..1da5b70 100644 --- a/spvwallet/cli/account/account.go +++ b/spvwallet/client/account/account.go @@ -1,17 +1,16 @@ package account import ( - "os" - "fmt" "errors" - "strings" + "fmt" "io/ioutil" + "os" + "strings" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/crypto" "github.com/elastos/Elastos.ELA.SPV/log" - . "github.com/elastos/Elastos.ELA.SPV/spvwallet" . "github.com/elastos/Elastos.ELA.SPV/spvwallet/cli" + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/crypto" "github.com/urfave/cli" ) diff --git a/spvwallet/cli/common.go b/spvwallet/client/common.go similarity index 91% rename from spvwallet/cli/common.go rename to spvwallet/client/common.go index 98a1ac3..75b00df 100644 --- a/spvwallet/cli/common.go +++ b/spvwallet/client/common.go @@ -1,15 +1,14 @@ package cli import ( - "os" - "fmt" "bufio" "errors" - "strings" + "fmt" + "os" "strconv" + "strings" - walt "github.com/elastos/Elastos.ELA.SPV/spvwallet" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/db" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/howeyc/gopass" @@ -61,7 +60,7 @@ func ShowAccountInfo(password []byte) error { return err } - keyStore, err := walt.OpenKeystore(password) + keyStore, err := OpenKeystore(password) if err != nil { return err } @@ -87,7 +86,7 @@ func ShowAccountInfo(password []byte) error { return nil } -func SelectAccount(wallet walt.Wallet) (string, error) { +func SelectAccount(wallet Wallet) (string, error) { addrs, err := wallet.GetAddrs() if err != nil || len(addrs) == 0 { return "", errors.New("fail to load wallet addresses") @@ -115,12 +114,12 @@ func SelectAccount(wallet walt.Wallet) (string, error) { return addrs[index].String(), nil } -func ShowAccounts(addrs []*db.Addr, newAddr *common.Uint168, wallet walt.Wallet) error { +func ShowAccounts(addrs []*util.Addr, newAddr *common.Uint168, wallet Wallet) error { // print header fmt.Printf("%5s %34s %-20s%22s %6s\n", "INDEX", "ADDRESS", "BALANCE", "(LOCKED)", "TYPE") fmt.Println("-----", strings.Repeat("-", 34), strings.Repeat("-", 42), "------") - currentHeight := wallet.ChainHeight() + currentHeight := wallet.BestHeight() for i, addr := range addrs { available := common.Fixed64(0) locked := common.Fixed64(0) diff --git a/spvwallet/client/database/database.go b/spvwallet/client/database/database.go new file mode 100644 index 0000000..90d79ad --- /dev/null +++ b/spvwallet/client/database/database.go @@ -0,0 +1,110 @@ +package database + +import ( + "sync" + + "github.com/elastos/Elastos.ELA.SPV/spvwallet/store" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/sqlite" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +type Database interface { + AddAddress(address *common.Uint168, script []byte, addrType int) error + GetAddress(address *common.Uint168) (*util.Addr, error) + GetAddrs() ([]*util.Addr, error) + DeleteAddress(address *common.Uint168) error + GetAddressUTXOs(address *common.Uint168) ([]*util.UTXO, error) + GetAddressSTXOs(address *common.Uint168) ([]*util.STXO, error) + BestHeight() uint32 + Clear() error +} + +func New() (Database, error) { + dataStore, err := sqlite.New() + if err != nil { + return nil, err + } + + return &database{ + lock: new(sync.RWMutex), + store: dataStore, + }, nil +} + +type database struct { + lock *sync.RWMutex + store store.DataStore +} + +func (d *database) AddAddress(address *common.Uint168, script []byte, addrType int) error { + d.lock.Lock() + defer d.lock.Unlock() + + return d.store.Addrs().Put(address, script, addrType) +} + +func (d *database) GetAddress(address *common.Uint168) (*util.Addr, error) { + d.lock.RLock() + defer d.lock.RUnlock() + + return d.store.Addrs().Get(address) +} + +func (d *database) GetAddrs() ([]*util.Addr, error) { + d.lock.RLock() + defer d.lock.RUnlock() + + return d.store.Addrs().GetAll() +} + +func (d *database) DeleteAddress(address *common.Uint168) error { + d.lock.Lock() + defer d.lock.Unlock() + + return d.store.Addrs().Delete(address) +} + +func (d *database) GetAddressUTXOs(address *common.Uint168) ([]*util.UTXO, error) { + d.lock.RLock() + defer d.lock.RUnlock() + + return d.store.UTXOs().GetAddrAll(address) +} + +func (d *database) GetAddressSTXOs(address *common.Uint168) ([]*util.STXO, error) { + d.lock.RLock() + defer d.lock.RUnlock() + + return d.store.STXOs().GetAddrAll(address) +} + +func (d *database) BestHeight() uint32 { + d.lock.RLock() + defer d.lock.RUnlock() + + return d.store.Chain().GetHeight() +} + +func (d *database) Clear() error { + d.lock.Lock() + defer d.lock.Unlock() + + headers, err := database.NewHeadersDB() + if err != nil { + return err + } + + err = headers.Clear() + if err != nil { + return err + } + + err = d.store.Clear() + if err != nil { + return err + } + + return nil +} diff --git a/spvwallet/client/interface.go b/spvwallet/client/interface.go new file mode 100644 index 0000000..cc83967 --- /dev/null +++ b/spvwallet/client/interface.go @@ -0,0 +1,382 @@ +package cli + +import ( + "bytes" + "errors" + "math" + "math/rand" + "strconv" + + "github.com/elastos/Elastos.ELA.SPV/log" + "github.com/elastos/Elastos.ELA.SPV/sdk" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/cli/database" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/crypto" + "github.com/elastos/Elastos.ELA/core" +) + +var SystemAssetId = getSystemAssetId() + +type Transfer struct { + Address string + Value *common.Fixed64 +} + +type Wallet interface { + database.Database + + VerifyPassword(password []byte) error + ChangePassword(oldPassword, newPassword []byte) error + + NewSubAccount(password []byte) (*common.Uint168, error) + AddMultiSignAccount(M uint, publicKey ...*crypto.PublicKey) (*common.Uint168, error) + + CreateTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64) (*core.Transaction, error) + CreateLockedTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64, lockedUntil uint32) (*core.Transaction, error) + CreateMultiOutputTransaction(fromAddress string, fee *common.Fixed64, output ...*Transfer) (*core.Transaction, error) + CreateLockedMultiOutputTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, output ...*Transfer) (*core.Transaction, error) + Sign(password []byte, transaction *core.Transaction) (*core.Transaction, error) + SendTransaction(txn *core.Transaction) error +} + +type wallet struct { + database.Database + Keystore +} + +func Create(password []byte) error { + keyStore, err := CreateKeystore(password) + if err != nil { + log.Error("Wallet create keystore failed:", err) + return err + } + + database, err := database.New() + if err != nil { + log.Error("Wallet create database failed:", err) + return err + } + + mainAccount := keyStore.GetAccountByIndex(0) + return database.AddAddress(mainAccount.ProgramHash(), + mainAccount.RedeemScript(), util.TypeMaster) +} + +func Open() (Wallet, error) { + database, err := database.New() + if err != nil { + log.Error("Wallet open database failed:", err) + return nil, err + } + + return &wallet{ + Database: database, + }, nil +} + +func (wallet *wallet) VerifyPassword(password []byte) error { + keyStore, err := OpenKeystore(password) + if err != nil { + return err + } + wallet.Keystore = keyStore + return nil +} + +func (wallet *wallet) NewSubAccount(password []byte) (*common.Uint168, error) { + err := wallet.VerifyPassword(password) + if err != nil { + return nil, err + } + + account := wallet.Keystore.NewAccount() + err = wallet.AddAddress(account.ProgramHash(), account.RedeemScript(), util.TypeSub) + if err != nil { + return nil, err + } + + // Notify SPV service to reload bloom filter with the new address + rpc.GetClient().NotifyNewAddress(account.ProgramHash().Bytes()) + + return account.ProgramHash(), nil +} + +func (wallet *wallet) AddMultiSignAccount(M uint, publicKeys ...*crypto.PublicKey) (*common.Uint168, error) { + redeemScript, err := crypto.CreateMultiSignRedeemScript(M, publicKeys) + if err != nil { + return nil, errors.New("[Wallet], CreateStandardRedeemScript failed") + } + + programHash, err := crypto.ToProgramHash(redeemScript) + if err != nil { + return nil, errors.New("[Wallet], CreateMultiSignAddress failed") + } + + err = wallet.AddAddress(programHash, redeemScript, util.TypeMulti) + if err != nil { + return nil, err + } + + // Notify SPV service to reload bloom filter with the new address + rpc.GetClient().NotifyNewAddress(programHash.Bytes()) + + return programHash, nil +} + +func (wallet *wallet) CreateTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64) (*core.Transaction, error) { + return wallet.CreateLockedTransaction(fromAddress, toAddress, amount, fee, uint32(0)) +} + +func (wallet *wallet) CreateLockedTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64, lockedUntil uint32) (*core.Transaction, error) { + return wallet.CreateLockedMultiOutputTransaction(fromAddress, fee, lockedUntil, &Transfer{toAddress, amount}) +} + +func (wallet *wallet) CreateMultiOutputTransaction(fromAddress string, fee *common.Fixed64, outputs ...*Transfer) (*core.Transaction, error) { + return wallet.CreateLockedMultiOutputTransaction(fromAddress, fee, uint32(0), outputs...) +} + +func (wallet *wallet) CreateLockedMultiOutputTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, outputs ...*Transfer) (*core.Transaction, error) { + return wallet.createTransaction(fromAddress, fee, lockedUntil, outputs...) +} + +func (wallet *wallet) createTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, outputs ...*Transfer) (*core.Transaction, error) { + // Check if output is valid + if outputs == nil || len(outputs) == 0 { + return nil, errors.New("[Wallet], Invalid transaction target") + } + + // Check if from address is valid + spender, err := common.Uint168FromAddress(fromAddress) + if err != nil { + return nil, errors.New("[Wallet], Invalid spender address") + } + // Create transaction outputs + var totalOutputValue = common.Fixed64(0) // The total value will be spend + var txOutputs []*core.Output // The outputs in transaction + totalOutputValue += *fee // Add transaction fee + + for _, output := range outputs { + receiver, err := common.Uint168FromAddress(output.Address) + if err != nil { + return nil, errors.New("[Wallet], Invalid receiver address") + } + txOutput := &core.Output{ + AssetID: SystemAssetId, + ProgramHash: *receiver, + Value: *output.Value, + OutputLock: lockedUntil, + } + totalOutputValue += *output.Value + txOutputs = append(txOutputs, txOutput) + } + // Get spender's UTXOs + utxos, err := wallet.GetAddressUTXOs(spender) + if err != nil { + return nil, errors.New("[Wallet], Get spender's UTXOs failed") + } + availableUTXOs := wallet.removeLockedUTXOs(utxos) // Remove locked UTXOs + availableUTXOs = util.SortUTXOs(availableUTXOs) // Sort available UTXOs by value ASC + + // Create transaction inputs + var txInputs []*core.Input // The inputs in transaction + for _, utxo := range availableUTXOs { + txInputs = append(txInputs, InputFromUTXO(utxo)) + if utxo.Value < totalOutputValue { + totalOutputValue -= utxo.Value + } else if utxo.Value == totalOutputValue { + totalOutputValue = 0 + break + } else if utxo.Value > totalOutputValue { + change := &core.Output{ + AssetID: SystemAssetId, + Value: utxo.Value - totalOutputValue, + OutputLock: uint32(0), + ProgramHash: *spender, + } + txOutputs = append(txOutputs, change) + totalOutputValue = 0 + break + } + } + if totalOutputValue > 0 { + return nil, errors.New("[Wallet], Available token is not enough") + } + + addr, err := wallet.GetAddress(spender) + if err != nil { + return nil, errors.New("[Wallet], Get spenders redeem script failed") + } + + return wallet.newTransaction(addr.Script(), txInputs, txOutputs), nil +} + +func (wallet *wallet) Sign(password []byte, txn *core.Transaction) (*core.Transaction, error) { + // Verify password + err := wallet.VerifyPassword(password) + if err != nil { + return nil, err + } + // Get sign type + signType, err := crypto.GetScriptType(txn.Programs[0].Code) + if err != nil { + return nil, err + } + // Look up transaction type + if signType == common.STANDARD { + + // Sign single transaction + txn, err = wallet.signStandardTransaction(txn) + if err != nil { + return nil, err + } + + } else if signType == common.MULTISIG { + + // Sign multi sign transaction + txn, err = wallet.signMultiSigTransaction(txn) + if err != nil { + return nil, err + } + } + + return txn, nil +} + +func (wallet *wallet) signStandardTransaction(txn *core.Transaction) (*core.Transaction, error) { + code := txn.Programs[0].Code + // Get signer + programHash, err := crypto.GetSigner(code) + // Check if current user is a valid signer + account := wallet.Keystore.GetAccountByProgramHash(programHash) + if account == nil { + return nil, errors.New("[Wallet], Invalid signer") + } + // Sign transaction + buf := new(bytes.Buffer) + txn.SerializeUnsigned(buf) + signature, err := account.Sign(buf.Bytes()) + if err != nil { + return nil, err + } + // Add signature + buf = new(bytes.Buffer) + buf.WriteByte(byte(len(signature))) + buf.Write(signature) + // Set program + var program = &core.Program{code, buf.Bytes()} + txn.Programs = []*core.Program{program} + + return txn, nil +} + +func (wallet *wallet) signMultiSigTransaction(txn *core.Transaction) (*core.Transaction, error) { + code := txn.Programs[0].Code + param := txn.Programs[0].Parameter + // Check if current user is a valid signer + var signerIndex = -1 + programHashes, err := crypto.GetSigners(code) + if err != nil { + return nil, err + } + var account *sdk.Account + for i, programHash := range programHashes { + account = wallet.Keystore.GetAccountByProgramHash(programHash) + if account != nil { + signerIndex = i + break + } + } + if signerIndex == -1 { + return nil, errors.New("[Wallet], Invalid multi sign signer") + } + // Sign transaction + buf := new(bytes.Buffer) + txn.SerializeUnsigned(buf) + signedTx, err := account.Sign(buf.Bytes()) + if err != nil { + return nil, err + } + // Append signature + txn.Programs[0].Parameter, err = crypto.AppendSignature(signerIndex, signedTx, buf.Bytes(), code, param) + if err != nil { + return nil, err + } + + return txn, nil +} + +func (wallet *wallet) SendTransaction(txn *core.Transaction) error { + // Send transaction through P2P network + return rpc.GetClient().SendTransaction(txn) +} + +func getSystemAssetId() common.Uint256 { + systemToken := &core.Transaction{ + TxType: core.RegisterAsset, + PayloadVersion: 0, + Payload: &core.PayloadRegisterAsset{ + Asset: core.Asset{ + Name: "ELA", + Precision: 0x08, + AssetType: 0x00, + }, + Amount: 0 * 100000000, + Controller: common.Uint168{}, + }, + Attributes: []*core.Attribute{}, + Inputs: []*core.Input{}, + Outputs: []*core.Output{}, + Programs: []*core.Program{}, + } + return systemToken.Hash() +} + +func (wallet *wallet) removeLockedUTXOs(utxos []*util.UTXO) []*util.UTXO { + var availableUTXOs []*util.UTXO + var currentHeight = wallet.BestHeight() + for _, utxo := range utxos { + if utxo.AtHeight == 0 { // remove unconfirmed UTOXs + continue + } + if utxo.LockTime > 0 { + if utxo.LockTime > currentHeight { + continue + } + utxo.LockTime = math.MaxUint32 - 1 + } + availableUTXOs = append(availableUTXOs, utxo) + } + return availableUTXOs +} + +func InputFromUTXO(utxo *util.UTXO) *core.Input { + input := new(core.Input) + input.Previous.TxID = utxo.Op.TxID + input.Previous.Index = utxo.Op.Index + input.Sequence = utxo.LockTime + return input +} + +func (wallet *wallet) newTransaction(redeemScript []byte, inputs []*core.Input, outputs []*core.Output) *core.Transaction { + // Create payload + txPayload := &core.PayloadTransferAsset{} + // Create attributes + txAttr := core.NewAttribute(core.Nonce, []byte(strconv.FormatInt(rand.Int63(), 10))) + attributes := make([]*core.Attribute, 0) + attributes = append(attributes, &txAttr) + // Create program + var program = &core.Program{redeemScript, nil} + // Create transaction + return &core.Transaction{ + TxType: core.TransferAsset, + Payload: txPayload, + Attributes: attributes, + Inputs: inputs, + Outputs: outputs, + Programs: []*core.Program{program}, + LockTime: wallet.BestHeight(), + } +} diff --git a/spvwallet/keystore.go b/spvwallet/client/keystore.go similarity index 100% rename from spvwallet/keystore.go rename to spvwallet/client/keystore.go diff --git a/spvwallet/keystore_file.go b/spvwallet/client/keystore_file.go similarity index 100% rename from spvwallet/keystore_file.go rename to spvwallet/client/keystore_file.go diff --git a/spvwallet/cli/transaction/transaction.go b/spvwallet/client/transaction/transaction.go similarity index 94% rename from spvwallet/cli/transaction/transaction.go rename to spvwallet/client/transaction/transaction.go index e9f2819..0edb041 100644 --- a/spvwallet/cli/transaction/transaction.go +++ b/spvwallet/client/transaction/transaction.go @@ -1,26 +1,25 @@ package transaction import ( - "os" - "fmt" - "bytes" "bufio" + "bytes" "errors" - "strings" - "strconv" + "fmt" "io/ioutil" + "os" + "strconv" + "strings" "github.com/elastos/Elastos.ELA.SPV/log" . "github.com/elastos/Elastos.ELA.SPV/spvwallet/cli" - walt "github.com/elastos/Elastos.ELA.SPV/spvwallet" - "github.com/elastos/Elastos.ELA/core" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" + "github.com/elastos/Elastos.ELA/core" "github.com/urfave/cli" ) -func CreateTransaction(c *cli.Context, wallet walt.Wallet) error { +func CreateTransaction(c *cli.Context, wallet Wallet) error { txn, err := createTransaction(c, wallet) if err != nil { return err @@ -28,7 +27,7 @@ func CreateTransaction(c *cli.Context, wallet walt.Wallet) error { return output(txn) } -func createTransaction(c *cli.Context, wallet walt.Wallet) (*core.Transaction, error) { +func createTransaction(c *cli.Context, wallet Wallet) (*core.Transaction, error) { feeStr := c.String("fee") if feeStr == "" { return nil, errors.New("use --fee to specify transfer fee") @@ -93,7 +92,7 @@ func createTransaction(c *cli.Context, wallet walt.Wallet) (*core.Transaction, e return txn, nil } -func createMultiOutputTransaction(c *cli.Context, wallet walt.Wallet, path, from string, fee *common.Fixed64) (*core.Transaction, error) { +func createMultiOutputTransaction(c *cli.Context, wallet Wallet, path, from string, fee *common.Fixed64) (*core.Transaction, error) { if _, err := os.Stat(path); err != nil { return nil, errors.New("invalid multi output file path") } @@ -103,7 +102,7 @@ func createMultiOutputTransaction(c *cli.Context, wallet walt.Wallet, path, from } scanner := bufio.NewScanner(file) - var multiOutput []*walt.Transfer + var multiOutput []*Transfer for scanner.Scan() { columns := strings.Split(scanner.Text(), ",") if len(columns) < 2 { @@ -115,7 +114,7 @@ func createMultiOutputTransaction(c *cli.Context, wallet walt.Wallet, path, from return nil, errors.New("invalid multi output transaction amount: " + amountStr) } address := strings.TrimSpace(columns[0]) - multiOutput = append(multiOutput, &walt.Transfer{address, amount}) + multiOutput = append(multiOutput, &Transfer{address, amount}) log.Trace("Multi output address:", address, ", amount:", amountStr) } @@ -140,7 +139,7 @@ func createMultiOutputTransaction(c *cli.Context, wallet walt.Wallet, path, from return txn, nil } -func SignTransaction(password []byte, context *cli.Context, wallet walt.Wallet) error { +func SignTransaction(password []byte, context *cli.Context, wallet Wallet) error { txn, err := getTransaction(context) if err != nil { return err @@ -154,7 +153,7 @@ func SignTransaction(password []byte, context *cli.Context, wallet walt.Wallet) return output(txn) } -func signTransaction(password []byte, wallet walt.Wallet, txn *core.Transaction) (*core.Transaction, error) { +func signTransaction(password []byte, wallet Wallet, txn *core.Transaction) (*core.Transaction, error) { haveSign, needSign, err := crypto.GetSignStatus(txn.Programs[0].Code, txn.Programs[0].Parameter) if haveSign == needSign { return nil, errors.New("transaction was fully signed, no need more sign") @@ -168,7 +167,7 @@ func signTransaction(password []byte, wallet walt.Wallet, txn *core.Transaction) return wallet.Sign(password, txn) } -func SendTransaction(password []byte, context *cli.Context, wallet walt.Wallet) error { +func SendTransaction(password []byte, context *cli.Context, wallet Wallet) error { content, err := getContent(context) var txn *core.Transaction @@ -301,7 +300,7 @@ func transactionAction(context *cli.Context) { } pass := context.String("password") - wallet, err := walt.Open() + wallet, err := Open() if err != nil { fmt.Println("error: open wallet failed,", err) os.Exit(2) diff --git a/spvwallet/cli/wallet/wallet.go b/spvwallet/client/wallet/wallet.go similarity index 98% rename from spvwallet/cli/wallet/wallet.go rename to spvwallet/client/wallet/wallet.go index c5fa5a6..4576076 100644 --- a/spvwallet/cli/wallet/wallet.go +++ b/spvwallet/client/wallet/wallet.go @@ -19,7 +19,7 @@ func createWallet(context *cli.Context) { return } - _, err = Create(password) + err = Create(password) if err != nil { fmt.Println("--CREAT WALLET FAILED--") return @@ -88,7 +88,7 @@ func resetDatabase(context *cli.Context) { return } - err = wallet.Reset() + err = wallet.Clear() if err != nil { fmt.Println("--WALLET DATABASE RESET FAILED--") return diff --git a/spvwallet/database.go b/spvwallet/database.go deleted file mode 100644 index 7e4b546..0000000 --- a/spvwallet/database.go +++ /dev/null @@ -1,113 +0,0 @@ -package spvwallet - -import ( - "sync" - - . "github.com/elastos/Elastos.ELA.SPV/spvwallet/db" - "github.com/elastos/Elastos.ELA.Utility/common" -) - -type Database interface { - AddAddress(address *common.Uint168, script []byte, addrType int) error - GetAddress(address *common.Uint168) (*Addr, error) - GetAddrs() ([]*Addr, error) - DeleteAddress(address *common.Uint168) error - GetAddressUTXOs(address *common.Uint168) ([]*UTXO, error) - GetAddressSTXOs(address *common.Uint168) ([]*STXO, error) - ChainHeight() uint32 - Reset() error -} - -var instance Database - -func GetDatabase() (Database, error) { - if instance == nil { - dataStore, err := NewSQLiteDB() - if err != nil { - return nil, err - } - - instance = &DatabaseImpl{ - lock: new(sync.RWMutex), - DataStore: dataStore, - } - } - - return instance, nil -} - -type DatabaseImpl struct { - lock *sync.RWMutex - DataStore -} - -func (db *DatabaseImpl) AddAddress(address *common.Uint168, script []byte, addrType int) error { - db.lock.Lock() - defer db.lock.Unlock() - - return db.DataStore.Addrs().Put(address, script, addrType) -} - -func (db *DatabaseImpl) GetAddress(address *common.Uint168) (*Addr, error) { - db.lock.RLock() - defer db.lock.RUnlock() - - return db.DataStore.Addrs().Get(address) -} - -func (db *DatabaseImpl) GetAddrs() ([]*Addr, error) { - db.lock.RLock() - defer db.lock.RUnlock() - - return db.DataStore.Addrs().GetAll() -} - -func (db *DatabaseImpl) DeleteAddress(address *common.Uint168) error { - db.lock.Lock() - defer db.lock.Unlock() - - return db.DataStore.Addrs().Delete(address) -} - -func (db *DatabaseImpl) GetAddressUTXOs(address *common.Uint168) ([]*UTXO, error) { - db.lock.RLock() - defer db.lock.RUnlock() - - return db.DataStore.UTXOs().GetAddrAll(address) -} - -func (db *DatabaseImpl) GetAddressSTXOs(address *common.Uint168) ([]*STXO, error) { - db.lock.RLock() - defer db.lock.RUnlock() - - return db.DataStore.STXOs().GetAddrAll(address) -} - -func (db *DatabaseImpl) ChainHeight() uint32 { - db.lock.RLock() - defer db.lock.RUnlock() - - return db.DataStore.Chain().GetHeight() -} - -func (db *DatabaseImpl) Reset() error { - db.lock.Lock() - defer db.lock.Unlock() - - headers, err := NewHeadersDB() - if err != nil { - return err - } - - err = headers.Reset() - if err != nil { - return err - } - - err = db.DataStore.Reset() - if err != nil { - return err - } - - return nil -} diff --git a/spvwallet/db/chain.go b/spvwallet/db/chain.go deleted file mode 100644 index a4bef51..0000000 --- a/spvwallet/db/chain.go +++ /dev/null @@ -1,52 +0,0 @@ -package db - -import ( - "database/sql" - "sync" -) - -const CreateInfoDB = `CREATE TABLE IF NOT EXISTS Chain( - Key NOT NULL PRIMARY KEY, - Value BLOB NOT NULL - );` - -const ( - HeightKey = "Height" -) - -type ChainDB struct { - *sync.RWMutex - *sql.DB -} - -func NewChainDB(db *sql.DB, lock *sync.RWMutex) (*ChainDB, error) { - _, err := db.Exec(CreateInfoDB) - if err != nil { - return nil, err - } - return &ChainDB{RWMutex: lock, DB: db}, nil -} - -// get chain height -func (db *ChainDB) GetHeight() uint32 { - db.RLock() - defer db.RUnlock() - - row := db.QueryRow("SELECT Value FROM Chain WHERE Key=?", HeightKey) - - var height uint32 - err := row.Scan(&height) - if err != nil { - return 0 - } - - return height -} - -// save chain height -func (db *ChainDB) PutHeight(height uint32) { - db.Lock() - defer db.Unlock() - - db.Exec("INSERT OR REPLACE INTO Chain(Key, Value) VALUES(?,?)", HeightKey, height) -} diff --git a/spvwallet/db/datastore.go b/spvwallet/db/datastore.go deleted file mode 100644 index b84e95c..0000000 --- a/spvwallet/db/datastore.go +++ /dev/null @@ -1,94 +0,0 @@ -package db - -import ( - "github.com/elastos/Elastos.ELA/core" - "github.com/elastos/Elastos.ELA.Utility/common" -) - -type DataStore interface { - Chain() Chain - Addrs() Addrs - Txs() Txs - UTXOs() UTXOs - STXOs() STXOs - - Rollback(height uint32) error - RollbackTx(txId *common.Uint256) error - // Reset database, clear all data - Reset() error - - Close() -} - -type Chain interface { - // save chain height - PutHeight(height uint32) - - // get chain height - GetHeight() uint32 -} - -type Addrs interface { - // put a address to database - Put(hash *common.Uint168, script []byte, addrType int) error - - // get a address from database - Get(hash *common.Uint168) (*Addr, error) - - // get all addresss from database - GetAll() ([]*Addr, error) - - // delete a address from database - Delete(hash *common.Uint168) error -} - -type Txs interface { - // Put a new transaction to database - Put(txn *Tx) error - - // Fetch a raw tx and it's metadata given a hash - Get(txId *common.Uint256) (*Tx, error) - - // Fetch all transactions from database - GetAll() ([]*Tx, error) - - // Fetch all transactions from the given height - GetAllFrom(height uint32) ([]*Tx, error) - - // Delete a transaction from the db - Delete(txId *common.Uint256) error -} - -type UTXOs interface { - // put a utxo to database - Put(hash *common.Uint168, utxo *UTXO) error - - // get a utxo from database - Get(outPoint *core.OutPoint) (*UTXO, error) - - // get utxos of the given address hash from database - GetAddrAll(hash *common.Uint168) ([]*UTXO, error) - - // Get all UTXOs in database - GetAll() ([]*UTXO, error) - - // delete a utxo from database - Delete(outPoint *core.OutPoint) error -} - -type STXOs interface { - // Move a UTXO to STXO - FromUTXO(outPoint *core.OutPoint, spendTxId *common.Uint256, spendHeight uint32) error - - // get a stxo from database - Get(outPoint *core.OutPoint) (*STXO, error) - - // get stxos of the given address hash from database - GetAddrAll(hash *common.Uint168) ([]*STXO, error) - - // Get all STXOs in database - GetAll() ([]*STXO, error) - - // delete a stxo from database - Delete(outPoint *core.OutPoint) error -} diff --git a/spvwallet/spvwallet.go b/spvwallet/spvwallet.go deleted file mode 100644 index 2d004fb..0000000 --- a/spvwallet/spvwallet.go +++ /dev/null @@ -1,282 +0,0 @@ -package spvwallet - -import ( - "sync" - "time" - - "github.com/elastos/Elastos.ELA.SPV/log" - "github.com/elastos/Elastos.ELA.SPV/sdk" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/db" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" - - "github.com/elastos/Elastos.ELA.SPV/net" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA/core" -) - -const ( - MaxUnconfirmedTime = time.Minute * 30 - MaxTxIdCached = 1000 - MaxConnections = 12 -) - -func Init(clientId uint64, seeds []string) (*SPVWallet, error) { - var err error - wallet := new(SPVWallet) - - // Initialize headers db - wallet.headerStore, err = db.NewHeadersDB() - if err != nil { - return nil, err - } - - // Initialize wallet database - wallet.dataStore, err = db.NewSQLiteDB() - if err != nil { - return nil, err - } - - // Initialize txs cache - wallet.txIds = NewTxIdCache(MaxTxIdCached) - - // Create server peer config - serverPeerConfig := net.ServerPeerConfig{ - Magic: config.Values().Magic, - Version: p2p.EIP001Version, - PeerId: clientId, - Port: 0, - Seeds: seeds, - MinOutbound: MaxConnections, - MaxConnections: MaxConnections, - } - - // Initialize spv service - wallet.SPVService, err = sdk.GetSPVService(sdk.SPVServiceConfig{ - Server: net.NewServerPeer(serverPeerConfig), - Foundation: config.Values().Foundation, - HeaderStore: wallet.headerStore, - GetFilterData: wallet.GetFilterData, - CommitTx: wallet.CommitTx, - OnBlockCommitted: wallet.OnBlockCommitted, - OnRollback: wallet.OnRollback, - }) - if err != nil { - return nil, err - } - - // Initialize RPC server - server := rpc.InitServer() - server.NotifyNewAddress = wallet.NotifyNewAddress - server.SendTransaction = wallet.SPVService.SendTransaction - wallet.rpcServer = server - - return wallet, nil -} - -type DataListener interface { - OnNewBlock(block *msg.MerkleBlock, txs []*core.Transaction) - OnRollback(height uint32) -} - -type SPVWallet struct { - sync.Mutex - sdk.SPVService - rpcServer *rpc.Server - headerStore *db.HeadersDB - dataStore db.DataStore - txIds *TxIdCache - filter *sdk.AddrFilter -} - -func (wallet *SPVWallet) Start() { - wallet.SPVService.Start() - wallet.rpcServer.Start() -} - -func (wallet *SPVWallet) Stop() { - wallet.SPVService.Stop() - wallet.rpcServer.Close() -} - -func (wallet *SPVWallet) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { - utxos, _ := wallet.dataStore.UTXOs().GetAll() - stxos, _ := wallet.dataStore.STXOs().GetAll() - - outpoints := make([]*core.OutPoint, 0, len(utxos)+len(stxos)) - for _, utxo := range utxos { - outpoints = append(outpoints, &utxo.Op) - } - for _, stxo := range stxos { - outpoints = append(outpoints, &stxo.Op) - } - - return wallet.getAddrFilter().GetAddrs(), outpoints -} - -// Commit a transaction return if this is a false positive and error -func (wallet *SPVWallet) CommitTx(tx *core.Transaction, height uint32) (bool, error) { - txId := tx.Hash() - - sh, ok := wallet.txIds.Get(txId) - if ok && (sh > 0 || (sh == 0 && height == 0)) { - return false, nil - } - - // Do not check double spends when syncing - if wallet.SPVService.ChainState() == sdk.WAITING { - dubs, err := wallet.checkDoubleSpends(tx) - if err != nil { - return false, nil - } - if len(dubs) > 0 { - if height == 0 { - return false, nil - } else { - // Rollback any double spend transactions - for _, dub := range dubs { - if err := wallet.dataStore.RollbackTx(dub); err != nil { - return false, nil - } - } - } - } - } - - hits := 0 - // Save UTXOs - for index, output := range tx.Outputs { - // Filter address - if wallet.getAddrFilter().ContainAddr(output.ProgramHash) { - var lockTime uint32 - if tx.TxType == core.CoinBase { - lockTime = height + 100 - } - utxo := ToUTXO(txId, height, index, output.Value, lockTime) - err := wallet.dataStore.UTXOs().Put(&output.ProgramHash, utxo) - if err != nil { - return false, err - } - hits++ - } - } - - // Put spent UTXOs to STXOs - for _, input := range tx.Inputs { - // Try to move UTXO to STXO, if a UTXO in database was spent, it will be moved to STXO - err := wallet.dataStore.STXOs().FromUTXO(&input.Previous, &txId, height) - if err == nil { - hits++ - } - } - - // If no hits, no need to save transaction - if hits == 0 { - return true, nil - } - - // Save transaction - err := wallet.dataStore.Txs().Put(db.NewTx(*tx, height)) - if err != nil { - return false, err - } - - wallet.txIds.Add(txId, height) - - return false, nil -} - -func (wallet *SPVWallet) OnBlockCommitted(block *msg.MerkleBlock, txs []*core.Transaction) { - wallet.dataStore.Chain().PutHeight(block.Header.(*core.Header).Height) - - // Check unconfirmed transaction timeout - if wallet.ChainState() == sdk.WAITING { - // Get all unconfirmed transactions - txs, err := wallet.dataStore.Txs().GetAllFrom(0) - if err != nil { - log.Debugf("Get unconfirmed transactions failed, error %s", err.Error()) - return - } - now := time.Now() - for _, tx := range txs { - if now.After(tx.Timestamp.Add(MaxUnconfirmedTime)) { - err = wallet.dataStore.RollbackTx(&tx.TxId) - if err != nil { - log.Errorf("Rollback timeout transaction %s failed, error %s", tx.TxId.String(), err.Error()) - } - } - } - } -} - -// Rollback chain data on the given height -func (wallet *SPVWallet) OnRollback(height uint32) error { - return wallet.dataStore.Rollback(height) -} - -func ToUTXO(txId common.Uint256, height uint32, index int, value common.Fixed64, lockTime uint32) *db.UTXO { - utxo := new(db.UTXO) - utxo.Op = *core.NewOutPoint(txId, uint16(index)) - utxo.Value = value - utxo.LockTime = lockTime - utxo.AtHeight = height - return utxo -} - -func (wallet *SPVWallet) NotifyNewAddress(hash []byte) { - // Reload address filter to include new address - wallet.loadAddrFilter() - // Broadcast filterload message to connected peers - wallet.ReloadFilter() -} - -func (wallet *SPVWallet) getAddrFilter() *sdk.AddrFilter { - if wallet.filter == nil { - wallet.loadAddrFilter() - } - return wallet.filter -} - -func (wallet *SPVWallet) loadAddrFilter() *sdk.AddrFilter { - addrs, _ := wallet.dataStore.Addrs().GetAll() - wallet.filter = sdk.NewAddrFilter(nil) - for _, addr := range addrs { - wallet.filter.AddAddr(addr.Hash()) - } - return wallet.filter -} - -// checkDoubleSpends takes a transaction and compares it with -// all transactions in the db. It returns a slice of all txIds in the db -// which are double spent by the received tx. -func (wallet *SPVWallet) checkDoubleSpends(tx *core.Transaction) ([]*common.Uint256, error) { - var dubs []*common.Uint256 - txId := tx.Hash() - txs, err := wallet.dataStore.Txs().GetAll() - if err != nil { - return nil, err - } - for _, compTx := range txs { - // Skip coinbase transaction - if compTx.Data.IsCoinBaseTx() { - continue - } - // Skip duplicate transaction - compTxId := compTx.Data.Hash() - if compTxId.IsEqual(txId) { - continue - } - for _, txIn := range tx.Inputs { - for _, compIn := range compTx.Data.Inputs { - if txIn.Previous.IsEqual(compIn.Previous) { - // Found double spend - dubs = append(dubs, &compTxId) - break // back to txIn loop - } - } - } - } - return dubs, nil -} diff --git a/spvwallet/db/headers.go b/spvwallet/store/headers/database.go similarity index 58% rename from spvwallet/db/headers.go rename to spvwallet/store/headers/database.go index 1cceba1..c320034 100644 --- a/spvwallet/db/headers.go +++ b/spvwallet/store/headers/database.go @@ -1,25 +1,28 @@ -package db +package headers import ( - "errors" "encoding/hex" + "errors" "fmt" "math/big" "sync" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.SPV/store" + "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/log" + "github.com/elastos/Elastos.ELA.SPV/util" "github.com/boltdb/bolt" - "github.com/cevaris/ordered_map" + "github.com/elastos/Elastos.ELA.Utility/common" ) -// HeadersDB implements Headers using bolt DB +// Ensure HeadersDB implement headers interface +var _ database.Headers = (*HeadersDB)(nil) + +// Headers implements Headers using bolt DB type HeadersDB struct { *sync.RWMutex *bolt.DB - cache *HeaderCache + cache *cache } var ( @@ -58,12 +61,12 @@ func NewHeadersDB() (*HeadersDB, error) { } func (h *HeadersDB) initCache() { - best, err := h.GetBestHeader() + best, err := h.GetBest() if err != nil { return } h.cache.tip = best - headers := []*store.StoreHeader{best} + headers := []*util.Header{best} for i := 0; i < 99; i++ { sh, err := h.GetPrevious(best) if err != nil { @@ -72,16 +75,15 @@ func (h *HeadersDB) initCache() { headers = append(headers, sh) } for i := len(headers) - 1; i >= 0; i-- { - h.cache.Set(headers[i]) + h.cache.set(headers[i]) } } -// Add a new header to blockchain -func (h *HeadersDB) PutHeader(header *store.StoreHeader, newTip bool) error { +func (h *HeadersDB) Put(header *util.Header, newTip bool) error { h.Lock() defer h.Unlock() - h.cache.Set(header) + h.cache.set(header) if newTip { h.cache.tip = header } @@ -108,20 +110,18 @@ func (h *HeadersDB) PutHeader(header *store.StoreHeader, newTip bool) error { }) } -// Get previous block of the given header -func (h *HeadersDB) GetPrevious(header *store.StoreHeader) (*store.StoreHeader, error) { +func (h *HeadersDB) GetPrevious(header *util.Header) (*util.Header, error) { if header.Height == 1 { - return &store.StoreHeader{TotalWork: new(big.Int)}, nil + return &util.Header{TotalWork: new(big.Int)}, nil } - return h.GetHeader(&header.Previous) + return h.Get(&header.Previous) } -// Get full header with it's hash -func (h *HeadersDB) GetHeader(hash *common.Uint256) (header *store.StoreHeader, err error) { +func (h *HeadersDB) Get(hash *common.Uint256) (header *util.Header, err error) { h.RLock() defer h.RUnlock() - header, err = h.cache.Get(hash) + header, err = h.cache.get(hash) if err == nil { return header, nil } @@ -143,8 +143,7 @@ func (h *HeadersDB) GetHeader(hash *common.Uint256) (header *store.StoreHeader, return header, err } -// Get the header on chain tip -func (h *HeadersDB) GetBestHeader() (header *store.StoreHeader, err error) { +func (h *HeadersDB) GetBest() (header *util.Header, err error) { h.RLock() defer h.RUnlock() @@ -169,7 +168,7 @@ func (h *HeadersDB) GetBestHeader() (header *store.StoreHeader, err error) { return header, err } -func (h *HeadersDB) Reset() error { +func (h *HeadersDB) Clear() error { h.Lock() defer h.Unlock() @@ -184,19 +183,20 @@ func (h *HeadersDB) Reset() error { } // Close db -func (h *HeadersDB) Close() { +func (h *HeadersDB) Close() error { h.Lock() - h.DB.Close() + err := h.DB.Close() log.Debug("Headers DB closed") + return err } -func getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*store.StoreHeader, error) { +func getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*util.Header, error) { headerBytes := tx.Bucket(bucket).Get(key) if headerBytes == nil { return nil, errors.New(fmt.Sprintf("Header %s does not exist in database", hex.EncodeToString(key))) } - var header store.StoreHeader + var header util.Header err := header.Deserialize(headerBytes) if err != nil { return nil, err @@ -204,46 +204,3 @@ func getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*store.StoreHeader, erro return &header, nil } - -type HeaderCache struct { - sync.RWMutex - size int - tip *store.StoreHeader - headers *ordered_map.OrderedMap -} - -func newHeaderCache(size int) *HeaderCache { - return &HeaderCache{ - size: size, - headers: ordered_map.NewOrderedMap(), - } -} - -func (cache *HeaderCache) pop() { - iter := cache.headers.IterFunc() - k, ok := iter() - if ok { - cache.headers.Delete(k.Key) - } -} - -func (cache *HeaderCache) Set(header *store.StoreHeader) { - cache.Lock() - defer cache.Unlock() - - if cache.headers.Len() > cache.size { - cache.pop() - } - cache.headers.Set(header.Hash().String(), header) -} - -func (cache *HeaderCache) Get(hash *common.Uint256) (*store.StoreHeader, error) { - cache.RLock() - defer cache.RUnlock() - - sh, ok := cache.headers.Get(hash.String()) - if !ok { - return nil, errors.New("Header not found in cache ") - } - return sh.(*store.StoreHeader), nil -} diff --git a/spvwallet/db/addrsdb.go b/spvwallet/store/sqlite/addrs.go similarity index 71% rename from spvwallet/db/addrsdb.go rename to spvwallet/store/sqlite/addrs.go index ade1b98..8a0fda7 100644 --- a/spvwallet/db/addrsdb.go +++ b/spvwallet/store/sqlite/addrs.go @@ -1,9 +1,11 @@ -package db +package sqlite import ( "database/sql" "sync" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + "github.com/elastos/Elastos.ELA.Utility/common" ) @@ -13,21 +15,21 @@ const CreateAddrsDB = `CREATE TABLE IF NOT EXISTS Addrs( Type INTEGER NOT NULL );` -type AddrsDB struct { +type Addrs struct { *sync.RWMutex *sql.DB } -func NewAddrsDB(db *sql.DB, lock *sync.RWMutex) (*AddrsDB, error) { +func NewAddrs(db *sql.DB, lock *sync.RWMutex) (*Addrs, error) { _, err := db.Exec(CreateAddrsDB) if err != nil { return nil, err } - return &AddrsDB{RWMutex: lock, DB: db}, nil + return &Addrs{RWMutex: lock, DB: db}, nil } // put a script to database -func (db *AddrsDB) Put(hash *common.Uint168, script []byte, addrType int) error { +func (db *Addrs) Put(hash *common.Uint168, script []byte, addrType int) error { db.Lock() defer db.Unlock() @@ -41,7 +43,7 @@ func (db *AddrsDB) Put(hash *common.Uint168, script []byte, addrType int) error } // get a script from database -func (db *AddrsDB) Get(hash *common.Uint168) (*Addr, error) { +func (db *Addrs) Get(hash *common.Uint168) (*util.Addr, error) { db.RLock() defer db.RUnlock() @@ -53,15 +55,15 @@ func (db *AddrsDB) Get(hash *common.Uint168) (*Addr, error) { return nil, err } - return NewAddr(hash, script, addrType), nil + return util.NewAddr(hash, script, addrType), nil } // get all Addrs from database -func (db *AddrsDB) GetAll() ([]*Addr, error) { +func (db *Addrs) GetAll() ([]*util.Addr, error) { db.RLock() defer db.RUnlock() - var addrs []*Addr + var addrs []*util.Addr rows, err := db.Query("SELECT Hash, Script, Type FROM Addrs") if err != nil { return addrs, err @@ -80,14 +82,14 @@ func (db *AddrsDB) GetAll() ([]*Addr, error) { if err != nil { return addrs, err } - addrs = append(addrs, NewAddr(hash, script, addrType)) + addrs = append(addrs, util.NewAddr(hash, script, addrType)) } return addrs, nil } // delete a script from database -func (db *AddrsDB) Delete(hash *common.Uint168) error { +func (db *Addrs) Delete(hash *common.Uint168) error { db.Lock() defer db.Unlock() diff --git a/spvwallet/db/sqlitedb.go b/spvwallet/store/sqlite/database.go similarity index 55% rename from spvwallet/db/sqlitedb.go rename to spvwallet/store/sqlite/database.go index d6bebb0..ebe06d7 100644 --- a/spvwallet/db/sqlitedb.go +++ b/spvwallet/store/sqlite/database.go @@ -3,11 +3,14 @@ package db import ( "database/sql" "fmt" + "github.com/elastos/Elastos.ELA.SPV/util" "sync" + "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/log" - _ "github.com/mattn/go-sqlite3" + "github.com/elastos/Elastos.ELA.Utility/common" + _ "github.com/mattn/go-sqlite3" ) const ( @@ -15,11 +18,14 @@ const ( DBName = "./spv_wallet.db" ) +// Ensure SQLiteDB implement TxsDB interface. +var _ database.TxsDB = (*SQLiteDB)(nil) + type SQLiteDB struct { *sync.RWMutex *sql.DB - chain *ChainDB + state *StateDB addrs *AddrsDB txs *TxsDB utxos *UTXOsDB @@ -35,8 +41,8 @@ func NewSQLiteDB() (*SQLiteDB, error) { // Use the same lock lock := new(sync.RWMutex) - // Create chain db - chainDB, err := NewChainDB(db, lock) + // Create state db + stateDB, err := NewStateDB(db, lock) if err != nil { return nil, err } @@ -65,7 +71,7 @@ func NewSQLiteDB() (*SQLiteDB, error) { RWMutex: lock, DB: db, - chain: chainDB, + state: stateDB, addrs: addrsDB, utxos: utxosDB, stxos: stxosDB, @@ -74,7 +80,7 @@ func NewSQLiteDB() (*SQLiteDB, error) { } func (db *SQLiteDB) Chain() Chain { - return db.chain + return db.state } func (db *SQLiteDB) Addrs() Addrs { @@ -93,6 +99,134 @@ func (db *SQLiteDB) STXOs() STXOs { return db.stxos } +// checkDoubleSpends takes a transaction and compares it with +// all transactions in the db. It returns a slice of all txIds in the db +// which are double spent by the received tx. +func (wallet *SQLiteDB) checkDoubleSpends(tx *util.Tx) ([]*common.Uint256, error) { + var dubs []*common.Uint256 + txId := tx.Hash() + txs, err := wallet.dataStore.Txs().GetAll() + if err != nil { + return nil, err + } + for _, compTx := range txs { + // Skip coinbase transaction + if compTx.Data.IsCoinBaseTx() { + continue + } + // Skip duplicate transaction + compTxId := compTx.Data.Hash() + if compTxId.IsEqual(txId) { + continue + } + for _, txIn := range tx.Inputs { + for _, compIn := range compTx.Data.Inputs { + if txIn.Previous.IsEqual(compIn.Previous) { + // Found double spend + dubs = append(dubs, &compTxId) + break // back to txIn loop + } + } + } + } + return dubs, nil +} + +// Batch returns a TxBatch instance for transactions batch +// commit, this can get better performance when commit a bunch +// of transactions within a block. +func (db *SQLiteDB) Batch() database.TxBatch { + +} + +// CommitTx save a transaction into database, and return +// if it is a false positive and error. +func (db *SQLiteDB) CommitTx(tx *util.Tx) (bool, error) { + txId := tx.Hash() + height := tx.Height + + sh, ok := db.txIds.Get(txId) + if ok && (sh > 0 || (sh == 0 && height == 0)) { + return false, nil + } + + dubs, err := db.checkDoubleSpends(tx) + if err != nil { + return false, nil + } + if len(dubs) > 0 { + if height == 0 { + return false, nil + } else { + // Rollback any double spend transactions + for _, dub := range dubs { + if err := db.RollbackTx(dub); err != nil { + return false, nil + } + } + } + } + + hits := 0 + // Save UTXOs + for index, output := range tx.Outputs { + // Filter address + if wallet.getAddrFilter().ContainAddr(output.ProgramHash) { + var lockTime uint32 + if tx.TxType == core.CoinBase { + lockTime = height + 100 + } + utxo := ToUTXO(txId, height, index, output.Value, lockTime) + err := wallet.dataStore.UTXOs().Put(&output.ProgramHash, utxo) + if err != nil { + return false, err + } + hits++ + } + } + + // Put spent UTXOs to STXOs + for _, input := range tx.Inputs { + // Try to move UTXO to STXO, if a UTXO in database was spent, it will be moved to STXO + err := wallet.dataStore.STXOs().FromUTXO(&input.Previous, &txId, height) + if err == nil { + hits++ + } + } + + // If no hits, no need to save transaction + if hits == 0 { + return true, nil + } + + // Save transaction + err := wallet.dataStore.Txs().Put(db.NewTx(*tx, height)) + if err != nil { + return false, err + } + + wallet.txIds.Add(txId, height) + + return false, nil +} + +// HaveTx returns if the transaction already saved in database +// by it's id. +func (db *SQLiteDB) HaveTx(txId *common.Uint256) (bool, error) { + +} + +// GetTxs returns all transactions within the given height. +func (db *SQLiteDB) GetTxs(height uint32) ([]*util.Tx, error) { + +} + +// RemoveTxs delete all transactions on the given height. Return +// how many transactions are deleted from database. +func (db *SQLiteDB) RemoveTxs(height uint32) (int, error) { + +} + func (db *SQLiteDB) Rollback(height uint32) error { db.Lock() defer db.Unlock() @@ -145,7 +279,7 @@ func (db *SQLiteDB) rollbackTx(txId *common.Uint256) error { // Get unconfirmed STXOs rows, err := db.Query( - "SELECT OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight FROM STXOs WHERE SpendHeight=?",0) + "SELECT OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight FROM STXOs WHERE SpendHeight=?", 0) if err != nil { return err } @@ -213,7 +347,7 @@ func (db *SQLiteDB) rollbackTx(txId *common.Uint256) error { return tx.Commit() } -func (db *SQLiteDB) Reset() error { +func (db *SQLiteDB) Clear() error { tx, err := db.Begin() if err != nil { return err diff --git a/spvwallet/store/sqlite/interface.go b/spvwallet/store/sqlite/interface.go new file mode 100644 index 0000000..5f3fb60 --- /dev/null +++ b/spvwallet/store/sqlite/interface.go @@ -0,0 +1,144 @@ +package store + +import ( + "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" +) + +type DataStore interface { + State() State + Addrs() Addrs + Txs() Txs + UTXOs() UTXOs + STXOs() STXOs + Batch() DataBatch + Clear() error + Close() error +} + +type Batch interface { + Rollback() error + Commit() error +} + +type DataBatch interface { + Addrs() AddrsBatch + Txs() TxsBatch + UTXOs() UTXOsBatch + STXOs() STXOsBatch +} + +type State interface { + // save state height + PutHeight(height uint32) + + // get state height + GetHeight() uint32 +} + +type Addrs interface { + // put a address to database + Put(hash *common.Uint168, script []byte, addrType int) error + + // get a address from database + Get(hash *common.Uint168) (*util.Addr, error) + + // get all addresss from database + GetAll() ([]*util.Addr, error) + + // delete a address from database + Del(hash *common.Uint168) error +} + +type AddrsBatch interface { + Batch + + // put a address to database + Put(hash *common.Uint168, script []byte, addrType int) error + + // delete a address from database + Del(hash *common.Uint168) error +} + +type Txs interface { + // Put a new transaction to database + Put(txn *util.Tx) error + + // Fetch a raw tx and it's metadata given a hash + Get(txId *common.Uint256) (*util.Tx, error) + + // Fetch all transactions from database + GetAll() ([]*util.Tx, error) + + // Delete a transaction from the db + Del(txId *common.Uint256) error +} + +type TxsBatch interface { + Batch + + // Put a new transaction to database + Put(txn *util.Tx) error + + // Delete a transaction from the db + Del(txId *common.Uint256) error + + // Delete transactions on the given height. + DelAll(height uint32) error +} + +type UTXOs interface { + // put a utxo to database + Put(hash *common.Uint168, utxo *util.UTXO) error + + // get a utxo from database + Get(op *core.OutPoint) (*util.UTXO, error) + + // get utxos of the given address hash from database + GetAddrAll(hash *common.Uint168) ([]*util.UTXO, error) + + // Get all UTXOs in database + GetAll() ([]*util.UTXO, error) + + // delete a utxo from database + Del(outPoint *core.OutPoint) error +} + +type UTXOsBatch interface { + Batch + + // put a utxo to database + Put(hash *common.Uint168, utxo *util.UTXO) error + + // delete a utxo from database + Del(outPoint *core.OutPoint) error +} + +type STXOs interface { + // Put save a STXO into database + Put(stxo *util.STXO) error + + // get a stxo from database + Get(op *core.OutPoint) (*util.STXO, error) + + // get stxos of the given address hash from database + GetAddrAll(hash *common.Uint168) ([]*util.STXO, error) + + // Get all STXOs in database + GetAll() ([]*util.STXO, error) + + // delete a stxo from database + Del(outPoint *core.OutPoint) error +} + +type STXOsBatch interface { + Batch + + // Put save a STXO into database + Put(stxo *util.STXO) error + + // delete a stxo from database + Del(outPoint *core.OutPoint) error +} diff --git a/spvwallet/store/sqlite/state.go b/spvwallet/store/sqlite/state.go new file mode 100644 index 0000000..dbbe454 --- /dev/null +++ b/spvwallet/store/sqlite/state.go @@ -0,0 +1,52 @@ +package db + +import ( + "database/sql" + "sync" +) + +const CreateStateDB = `CREATE TABLE IF NOT EXISTS State( + Key NOT NULL PRIMARY KEY, + Value BLOB NOT NULL + );` + +const ( + HeightKey = "Height" +) + +type StateDB struct { + *sync.RWMutex + *sql.DB +} + +func NewStateDB(db *sql.DB, lock *sync.RWMutex) (*StateDB, error) { + _, err := db.Exec(CreateStateDB) + if err != nil { + return nil, err + } + return &StateDB{RWMutex: lock, DB: db}, nil +} + +// get state height +func (db *StateDB) GetHeight() uint32 { + db.RLock() + defer db.RUnlock() + + row := db.QueryRow("SELECT Value FROM State WHERE Key=?", HeightKey) + + var height uint32 + err := row.Scan(&height) + if err != nil { + return 0 + } + + return height +} + +// save state height +func (db *StateDB) PutHeight(height uint32) { + db.Lock() + defer db.Unlock() + + db.Exec("INSERT OR REPLACE INTO State(Key, Value) VALUES(?,?)", HeightKey, height) +} diff --git a/spvwallet/db/stxosdb.go b/spvwallet/store/sqlite/stxos.go similarity index 76% rename from spvwallet/db/stxosdb.go rename to spvwallet/store/sqlite/stxos.go index f5f8ab3..b78b704 100644 --- a/spvwallet/db/stxosdb.go +++ b/spvwallet/store/sqlite/stxos.go @@ -1,7 +1,8 @@ -package db +package sqlite import ( "database/sql" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" "sync" "fmt" @@ -19,21 +20,21 @@ const CreateSTXOsDB = `CREATE TABLE IF NOT EXISTS STXOs( ScriptHash BLOB NOT NULL );` -type STXOsDB struct { +type STXOs struct { *sync.RWMutex *sql.DB } -func NewSTXOsDB(db *sql.DB, lock *sync.RWMutex) (*STXOsDB, error) { +func NewSTXOs(db *sql.DB, lock *sync.RWMutex) (*STXOs, error) { _, err := db.Exec(CreateSTXOsDB) if err != nil { return nil, err } - return &STXOsDB{RWMutex: lock, DB: db}, nil + return &STXOs{RWMutex: lock, DB: db}, nil } // Move a UTXO to STXO -func (db *STXOsDB) FromUTXO(outPoint *core.OutPoint, spendTxId *common.Uint256, spendHeight uint32) error { +func (db *STXOs) FromUTXO(outPoint *core.OutPoint, spendTxId *common.Uint256, spendHeight uint32) error { db.Lock() defer db.Unlock() @@ -69,7 +70,7 @@ func (db *STXOsDB) FromUTXO(outPoint *core.OutPoint, spendTxId *common.Uint256, } // get a stxo from database -func (db *STXOsDB) Get(outPoint *core.OutPoint) (*STXO, error) { +func (db *STXOs) Get(outPoint *core.OutPoint) (*util.STXO, error) { db.RLock() defer db.RUnlock() @@ -91,17 +92,17 @@ func (db *STXOsDB) Get(outPoint *core.OutPoint) (*STXO, error) { return nil, err } - var utxo = UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight} + var utxo = util.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight} spendHash, err := common.Uint256FromBytes(spendHashBytes) if err != nil { return nil, err } - return &STXO{UTXO: utxo, SpendTxId: *spendHash, SpendHeight: spendHeight}, nil + return &util.STXO{UTXO: utxo, SpendTxId: *spendHash, SpendHeight: spendHeight}, nil } // get stxos of the given script hash from database -func (db *STXOsDB) GetAddrAll(hash *common.Uint168) ([]*STXO, error) { +func (db *STXOs) GetAddrAll(hash *common.Uint168) ([]*util.STXO, error) { db.RLock() defer db.RUnlock() @@ -115,7 +116,7 @@ func (db *STXOsDB) GetAddrAll(hash *common.Uint168) ([]*STXO, error) { return db.getSTXOs(rows) } -func (db *STXOsDB) GetAll() ([]*STXO, error) { +func (db *STXOs) GetAll() ([]*util.STXO, error) { db.RLock() defer db.RUnlock() @@ -128,8 +129,8 @@ func (db *STXOsDB) GetAll() ([]*STXO, error) { return db.getSTXOs(rows) } -func (db *STXOsDB) getSTXOs(rows *sql.Rows) ([]*STXO, error) { - var stxos []*STXO +func (db *STXOs) getSTXOs(rows *sql.Rows) ([]*util.STXO, error) { + var stxos []*util.STXO for rows.Next() { var opBytes []byte var valueBytes []byte @@ -151,20 +152,20 @@ func (db *STXOsDB) getSTXOs(rows *sql.Rows) ([]*STXO, error) { if err != nil { return stxos, err } - var utxo = UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight} + var utxo = util.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight} spendHash, err := common.Uint256FromBytes(spendHashBytes) if err != nil { return stxos, err } - stxos = append(stxos, &STXO{UTXO: utxo, SpendTxId: *spendHash, SpendHeight: spendHeight}) + stxos = append(stxos, &util.STXO{UTXO: utxo, SpendTxId: *spendHash, SpendHeight: spendHeight}) } return stxos, nil } // delete a stxo from database -func (db *STXOsDB) Delete(outPoint *core.OutPoint) error { +func (db *STXOs) Delete(outPoint *core.OutPoint) error { db.Lock() defer db.Unlock() diff --git a/spvwallet/db/txsdb.go b/spvwallet/store/sqlite/txs.go similarity index 64% rename from spvwallet/db/txsdb.go rename to spvwallet/store/sqlite/txs.go index 219e5be..aea3a09 100644 --- a/spvwallet/db/txsdb.go +++ b/spvwallet/store/sqlite/txs.go @@ -1,38 +1,40 @@ -package db +package sqlite import ( "bytes" "database/sql" "math" "sync" + "time" + + "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" - "github.com/elastos/Elastos.ELA/core" "github.com/elastos/Elastos.ELA.Utility/common" - "time" + "github.com/elastos/Elastos.ELA/core" ) -const CreateTXNDB = `CREATE TABLE IF NOT EXISTS TXNs( +const CreateTxsDB = `CREATE TABLE IF NOT EXISTS Txs( Hash BLOB NOT NULL PRIMARY KEY, Height INTEGER NOT NULL, Timestamp INTEGER NOT NULL, RawData BLOB NOT NULL );` -type TxsDB struct { +type TXs struct { *sync.RWMutex *sql.DB } -func NewTxsDB(db *sql.DB, lock *sync.RWMutex) (*TxsDB, error) { - _, err := db.Exec(CreateTXNDB) +func NewTxs(db *sql.DB, lock *sync.RWMutex) (*TXs, error) { + _, err := db.Exec(CreateTxsDB) if err != nil { return nil, err } - return &TxsDB{RWMutex: lock, DB: db}, nil + return &TXs{RWMutex: lock, DB: db}, nil } // Put a new transaction to database -func (t *TxsDB) Put(storeTx *Tx) error { +func (t *TXs) Put(storeTx *util.Tx) error { t.Lock() defer t.Unlock() @@ -42,7 +44,7 @@ func (t *TxsDB) Put(storeTx *Tx) error { return err } - sql := `INSERT OR REPLACE INTO TXNs(Hash, Height, Timestamp, RawData) VALUES(?,?,?,?)` + sql := `INSERT OR REPLACE INTO Txs(Hash, Height, Timestamp, RawData) VALUES(?,?,?,?)` _, err = t.Exec(sql, storeTx.TxId.Bytes(), storeTx.Height, storeTx.Timestamp.Unix(), buf.Bytes()) if err != nil { return err @@ -52,11 +54,11 @@ func (t *TxsDB) Put(storeTx *Tx) error { } // Fetch a raw tx and it's metadata given a hash -func (t *TxsDB) Get(txId *common.Uint256) (*Tx, error) { +func (t *TXs) Get(txId *common.Uint256) (*util.Tx, error) { t.RLock() defer t.RUnlock() - row := t.QueryRow(`SELECT Height, Timestamp, RawData FROM TXNs WHERE Hash=?`, txId.Bytes()) + row := t.QueryRow(`SELECT Height, Timestamp, RawData FROM Txs WHERE Hash=?`, txId.Bytes()) var height uint32 var timestamp int64 var rawData []byte @@ -70,24 +72,24 @@ func (t *TxsDB) Get(txId *common.Uint256) (*Tx, error) { return nil, err } - return &Tx{TxId: *txId, Height: height, Timestamp: time.Unix(timestamp, 0), Data: tx}, nil + return &util.Tx{TxId: *txId, Height: height, Timestamp: time.Unix(timestamp, 0), Data: tx}, nil } // Fetch all transactions from database -func (t *TxsDB) GetAll() ([]*Tx, error) { +func (t *TXs) GetAll() ([]*util.Tx, error) { return t.GetAllFrom(math.MaxUint32) } // Fetch all transactions from the given height -func (t *TxsDB) GetAllFrom(height uint32) ([]*Tx, error) { +func (t *TXs) GetAllFrom(height uint32) ([]*util.Tx, error) { t.RLock() defer t.RUnlock() - sql := "SELECT Hash, Height, Timestamp, RawData FROM TXNs" + sql := "SELECT Hash, Height, Timestamp, RawData FROM Txs" if height != math.MaxUint32 { sql += " WHERE Height=?" } - var txns []*Tx + var txns []*util.Tx rows, err := t.Query(sql, height) if err != nil { return txns, err @@ -115,18 +117,18 @@ func (t *TxsDB) GetAllFrom(height uint32) ([]*Tx, error) { return nil, err } - txns = append(txns, &Tx{TxId: *txId, Height: height, Timestamp: time.Unix(timestamp, 0), Data: tx}) + txns = append(txns, &util.Tx{TxId: *txId, Height: height, Timestamp: time.Unix(timestamp, 0), Data: tx}) } return txns, nil } // Delete a transaction from the db -func (t *TxsDB) Delete(txId *common.Uint256) error { +func (t *TXs) Delete(txId *common.Uint256) error { t.Lock() defer t.Unlock() - _, err := t.Exec("DELETE FROM TXNs WHERE Hash=?", txId.Bytes()) + _, err := t.Exec("DELETE FROM Txs WHERE Hash=?", txId.Bytes()) if err != nil { return err } diff --git a/spvwallet/db/utxosdb.go b/spvwallet/store/sqlite/utxos.go similarity index 74% rename from spvwallet/db/utxosdb.go rename to spvwallet/store/sqlite/utxos.go index 5122109..539c9ce 100644 --- a/spvwallet/db/utxosdb.go +++ b/spvwallet/store/sqlite/utxos.go @@ -1,9 +1,11 @@ -package db +package sqlite import ( "database/sql" "sync" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" ) @@ -16,21 +18,21 @@ const CreateUTXOsDB = `CREATE TABLE IF NOT EXISTS UTXOs( ScriptHash BLOB NOT NULL );` -type UTXOsDB struct { +type UTXOs struct { *sync.RWMutex *sql.DB } -func NewUTXOsDB(db *sql.DB, lock *sync.RWMutex) (*UTXOsDB, error) { +func NewUTXOs(db *sql.DB, lock *sync.RWMutex) (*UTXOs, error) { _, err := db.Exec(CreateUTXOsDB) if err != nil { return nil, err } - return &UTXOsDB{RWMutex: lock, DB: db}, nil + return &UTXOs{RWMutex: lock, DB: db}, nil } // put a utxo to database -func (db *UTXOsDB) Put(hash *common.Uint168, utxo *UTXO) error { +func (db *UTXOs) Put(hash *common.Uint168, utxo *util.UTXO) error { db.Lock() defer db.Unlock() @@ -48,7 +50,7 @@ func (db *UTXOsDB) Put(hash *common.Uint168, utxo *UTXO) error { } // get a utxo from database -func (db *UTXOsDB) Get(outPoint *core.OutPoint) (*UTXO, error) { +func (db *UTXOs) Get(outPoint *core.OutPoint) (*util.UTXO, error) { db.RLock() defer db.RUnlock() @@ -67,11 +69,11 @@ func (db *UTXOsDB) Get(outPoint *core.OutPoint) (*UTXO, error) { return nil, err } - return &UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight}, nil + return &util.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight}, nil } // get utxos of the given script hash from database -func (db *UTXOsDB) GetAddrAll(hash *common.Uint168) ([]*UTXO, error) { +func (db *UTXOs) GetAddrAll(hash *common.Uint168) ([]*util.UTXO, error) { db.RLock() defer db.RUnlock() @@ -85,21 +87,21 @@ func (db *UTXOsDB) GetAddrAll(hash *common.Uint168) ([]*UTXO, error) { return db.getUTXOs(rows) } -func (db *UTXOsDB) GetAll() ([]*UTXO, error) { +func (db *UTXOs) GetAll() ([]*util.UTXO, error) { db.RLock() defer db.RUnlock() rows, err := db.Query("SELECT OutPoint, Value, LockTime, AtHeight FROM UTXOs") if err != nil { - return []*UTXO{}, err + return []*util.UTXO{}, err } defer rows.Close() return db.getUTXOs(rows) } -func (db *UTXOsDB) getUTXOs(rows *sql.Rows) ([]*UTXO, error) { - var utxos []*UTXO +func (db *UTXOs) getUTXOs(rows *sql.Rows) ([]*util.UTXO, error) { + var utxos []*util.UTXO for rows.Next() { var opBytes []byte var valueBytes []byte @@ -119,14 +121,14 @@ func (db *UTXOsDB) getUTXOs(rows *sql.Rows) ([]*UTXO, error) { if err != nil { return utxos, err } - utxos = append(utxos, &UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight}) + utxos = append(utxos, &util.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight}) } return utxos, nil } // delete a utxo from database -func (db *UTXOsDB) Delete(outPoint *core.OutPoint) error { +func (db *UTXOs) Delete(outPoint *core.OutPoint) error { db.Lock() defer db.Unlock() diff --git a/spvwallet/db/addr.go b/spvwallet/sutil/addr.go similarity index 98% rename from spvwallet/db/addr.go rename to spvwallet/sutil/addr.go index 9119540..700e66d 100644 --- a/spvwallet/db/addr.go +++ b/spvwallet/sutil/addr.go @@ -1,4 +1,4 @@ -package db +package util import "github.com/elastos/Elastos.ELA.Utility/common" diff --git a/spvwallet/db/stxo.go b/spvwallet/sutil/stxo.go similarity index 98% rename from spvwallet/db/stxo.go rename to spvwallet/sutil/stxo.go index 8bcedcf..1f767a5 100644 --- a/spvwallet/db/stxo.go +++ b/spvwallet/sutil/stxo.go @@ -1,4 +1,4 @@ -package db +package util import ( "fmt" diff --git a/spvwallet/db/tx.go b/spvwallet/sutil/tx.go similarity index 98% rename from spvwallet/db/tx.go rename to spvwallet/sutil/tx.go index a85d3df..3008d32 100644 --- a/spvwallet/db/tx.go +++ b/spvwallet/sutil/tx.go @@ -1,4 +1,4 @@ -package db +package util import ( "time" diff --git a/spvwallet/db/utxo.go b/spvwallet/sutil/utxo.go similarity index 99% rename from spvwallet/db/utxo.go rename to spvwallet/sutil/utxo.go index 1ca8856..b17fe3d 100644 --- a/spvwallet/db/utxo.go +++ b/spvwallet/sutil/utxo.go @@ -1,4 +1,4 @@ -package db +package util import ( "fmt" diff --git a/spvwallet/wallet.go b/spvwallet/wallet.go index e7d9415..20b1f03 100644 --- a/spvwallet/wallet.go +++ b/spvwallet/wallet.go @@ -1,391 +1,307 @@ package spvwallet import ( - "bytes" - "errors" - "math" - "math/rand" - "strconv" + "time" + "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/log" "github.com/elastos/Elastos.ELA.SPV/sdk" - . "github.com/elastos/Elastos.ELA.SPV/spvwallet/db" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/db" "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/crypto" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA/core" ) -var SystemAssetId = getSystemAssetId() - -type Transfer struct { - Address string - Value *common.Fixed64 -} - -var wallet Wallet // Single instance of wallet - -type Wallet interface { - Database - - VerifyPassword(password []byte) error - ChangePassword(oldPassword, newPassword []byte) error - - NewSubAccount(password []byte) (*common.Uint168, error) - AddMultiSignAccount(M uint, publicKey ...*crypto.PublicKey) (*common.Uint168, error) - - CreateTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64) (*core.Transaction, error) - CreateLockedTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64, lockedUntil uint32) (*core.Transaction, error) - CreateMultiOutputTransaction(fromAddress string, fee *common.Fixed64, output ...*Transfer) (*core.Transaction, error) - CreateLockedMultiOutputTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, output ...*Transfer) (*core.Transaction, error) - Sign(password []byte, transaction *core.Transaction) (*core.Transaction, error) - SendTransaction(txn *core.Transaction) error -} +const ( + MaxUnconfirmedTime = time.Minute * 30 + MaxTxIdCached = 1000 + MaxPeers = 12 + MinPeersForSync = 2 +) -type WalletImpl struct { - Database - Keystore -} +func Init(clientId uint64, seeds []string) (*SPVWallet, error) { + wallet := new(SPVWallet) -func Create(password []byte) (Wallet, error) { - keyStore, err := CreateKeystore(password) + // Initialize headers db + headers, err := db.NewHeadersDB() if err != nil { - log.Error("Wallet create keystore failed:", err) return nil, err } - database, err := GetDatabase() + // Initialize wallet database + wallet.dataStore, err = db.NewSQLiteDB() if err != nil { - log.Error("Wallet create database failed:", err) return nil, err } - mainAccount := keyStore.GetAccountByIndex(0) - database.AddAddress(mainAccount.ProgramHash(), mainAccount.RedeemScript(), TypeMaster) - - wallet = &WalletImpl{ - Database: database, - Keystore: keyStore, + // Initiate ChainStore + wallet.chainStore = database.NewDefaultChainDB(headers, wallet) + + // Initialize txs cache + wallet.txIds = NewTxIdCache(MaxTxIdCached) + + // Initialize spv service + wallet.IService, err = sdk.NewService( + &sdk.Config{ + Magic: config.Values().Magic, + SeedList: config.Values().SeedList, + MaxPeers: MaxPeers, + MinPeersForSync: MinPeersForSync, + Foundation: config.Values().Foundation, + ChainStore: wallet.chainStore, + GetFilterData: wallet.GetFilterData, + }) + if err != nil { + return nil, err } - return wallet, nil -} -func Open() (Wallet, error) { - if wallet == nil { - database, err := GetDatabase() - if err != nil { - log.Error("Wallet open database failed:", err) - return nil, err - } + // Initialize RPC server + server := rpc.InitServer() + server.NotifyNewAddress = wallet.NotifyNewAddress + server.SendTransaction = wallet.IService.SendTransaction + wallet.rpcServer = server - wallet = &WalletImpl{ - Database: database, - } - } return wallet, nil } -func (wallet *WalletImpl) VerifyPassword(password []byte) error { - keyStore, err := OpenKeystore(password) - if err != nil { - return err - } - wallet.Keystore = keyStore - return nil +type DataListener interface { + OnNewBlock(block *msg.MerkleBlock, txs []*core.Transaction) + OnRollback(height uint32) } -func (wallet *WalletImpl) NewSubAccount(password []byte) (*common.Uint168, error) { - err := wallet.VerifyPassword(password) - if err != nil { - return nil, err - } - - account := wallet.Keystore.NewAccount() - err = wallet.AddAddress(account.ProgramHash(), account.RedeemScript(), TypeSub) - if err != nil { - return nil, err - } +type SPVWallet struct { + sdk.IService + rpcServer *rpc.Server + chainStore database.ChainStore + dataStore db.DataStore + txIds *TxIdCache + filter *sdk.AddrFilter +} - // Notify SPV service to reload bloom filter with the new address - rpc.GetClient().NotifyNewAddress(account.ProgramHash().Bytes()) +func (wallet *SPVWallet) Start() { + wallet.IService.Start() + wallet.rpcServer.Start() +} - return account.ProgramHash(), nil +func (wallet *SPVWallet) Stop() { + wallet.IService.Stop() + wallet.rpcServer.Close() } -func (wallet *WalletImpl) AddMultiSignAccount(M uint, publicKeys ...*crypto.PublicKey) (*common.Uint168, error) { - redeemScript, err := crypto.CreateMultiSignRedeemScript(M, publicKeys) - if err != nil { - return nil, errors.New("[Wallet], CreateStandardRedeemScript failed") - } +func (wallet *SPVWallet) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { + utxos, _ := wallet.dataStore.UTXOs().GetAll() + stxos, _ := wallet.dataStore.STXOs().GetAll() - programHash, err := crypto.ToProgramHash(redeemScript) - if err != nil { - return nil, errors.New("[Wallet], CreateMultiSignAddress failed") + outpoints := make([]*core.OutPoint, 0, len(utxos)+len(stxos)) + for _, utxo := range utxos { + outpoints = append(outpoints, &utxo.Op) } - - err = wallet.AddAddress(programHash, redeemScript, TypeMulti) - if err != nil { - return nil, err + for _, stxo := range stxos { + outpoints = append(outpoints, &stxo.Op) } - // Notify SPV service to reload bloom filter with the new address - rpc.GetClient().NotifyNewAddress(programHash.Bytes()) - - return programHash, nil + return wallet.getAddrFilter().GetAddrs(), outpoints } -func (wallet *WalletImpl) CreateTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64) (*core.Transaction, error) { - return wallet.CreateLockedTransaction(fromAddress, toAddress, amount, fee, uint32(0)) -} +func (wallet *SPVWallet) isFalsePositive(tx *core.Transaction) (bool, error) { -func (wallet *WalletImpl) CreateLockedTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64, lockedUntil uint32) (*core.Transaction, error) { - return wallet.CreateLockedMultiOutputTransaction(fromAddress, fee, lockedUntil, &Transfer{toAddress, amount}) -} + hits := 0 + // Save UTXOs + for index, output := range tx.Outputs { + // Filter address + if wallet.getAddrFilter().ContainAddr(output.ProgramHash) { + var lockTime uint32 + if tx.TxType == core.CoinBase { + lockTime = height + 100 + } + utxo := ToUTXO(txId, height, index, output.Value, lockTime) + err := wallet.dataStore.UTXOs().Put(&output.ProgramHash, utxo) + if err != nil { + return false, err + } + hits++ + } + } -func (wallet *WalletImpl) CreateMultiOutputTransaction(fromAddress string, fee *common.Fixed64, outputs ...*Transfer) (*core.Transaction, error) { - return wallet.CreateLockedMultiOutputTransaction(fromAddress, fee, uint32(0), outputs...) -} + // Put spent UTXOs to STXOs + for _, input := range tx.Inputs { + // Try to move UTXO to STXO, if a UTXO in database was spent, it will be moved to STXO + err := wallet.dataStore.STXOs().FromUTXO(&input.Previous, &txId, height) + if err == nil { + hits++ + } + } -func (wallet *WalletImpl) CreateLockedMultiOutputTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, outputs ...*Transfer) (*core.Transaction, error) { - return wallet.createTransaction(fromAddress, fee, lockedUntil, outputs...) + // If no hits, no need to save transaction + if hits == 0 { + return true, nil + } } -func (wallet *WalletImpl) createTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, outputs ...*Transfer) (*core.Transaction, error) { - // Check if output is valid - if outputs == nil || len(outputs) == 0 { - return nil, errors.New("[Wallet], Invalid transaction target") - } +// Commit a transaction return if this is a false positive and error +func (wallet *SPVWallet) CommitTx(tx *core.Transaction, height uint32) (bool, error) { + txId := tx.Hash() - // Check if from address is valid - spender, err := common.Uint168FromAddress(fromAddress) - if err != nil { - return nil, errors.New("[Wallet], Invalid spender address") + sh, ok := wallet.txIds.Get(txId) + if ok && (sh > 0 || (sh == 0 && height == 0)) { + return false, nil } - // Create transaction outputs - var totalOutputValue = common.Fixed64(0) // The total value will be spend - var txOutputs []*core.Output // The outputs in transaction - totalOutputValue += *fee // Add transaction fee - for _, output := range outputs { - receiver, err := common.Uint168FromAddress(output.Address) + // Do not check double spends when syncing + if wallet.IsCurrent() { + dubs, err := wallet.checkDoubleSpends(tx) if err != nil { - return nil, errors.New("[Wallet], Invalid receiver address") + return false, nil } - txOutput := &core.Output{ - AssetID: SystemAssetId, - ProgramHash: *receiver, - Value: *output.Value, - OutputLock: lockedUntil, + if len(dubs) > 0 { + if height == 0 { + return false, nil + } else { + // Rollback any double spend transactions + for _, dub := range dubs { + if err := wallet.dataStore.RollbackTx(dub); err != nil { + return false, nil + } + } + } } - totalOutputValue += *output.Value - txOutputs = append(txOutputs, txOutput) - } - // Get spender's UTXOs - utxos, err := wallet.GetAddressUTXOs(spender) - if err != nil { - return nil, errors.New("[Wallet], Get spender's UTXOs failed") } - availableUTXOs := wallet.removeLockedUTXOs(utxos) // Remove locked UTXOs - availableUTXOs = SortUTXOs(availableUTXOs) // Sort available UTXOs by value ASC - - // Create transaction inputs - var txInputs []*core.Input // The inputs in transaction - for _, utxo := range availableUTXOs { - txInputs = append(txInputs, InputFromUTXO(utxo)) - if utxo.Value < totalOutputValue { - totalOutputValue -= utxo.Value - } else if utxo.Value == totalOutputValue { - totalOutputValue = 0 - break - } else if utxo.Value > totalOutputValue { - change := &core.Output{ - AssetID: SystemAssetId, - Value: utxo.Value - totalOutputValue, - OutputLock: uint32(0), - ProgramHash: *spender, + + hits := 0 + // Save UTXOs + for index, output := range tx.Outputs { + // Filter address + if wallet.getAddrFilter().ContainAddr(output.ProgramHash) { + var lockTime uint32 + if tx.TxType == core.CoinBase { + lockTime = height + 100 } - txOutputs = append(txOutputs, change) - totalOutputValue = 0 - break + utxo := ToUTXO(txId, height, index, output.Value, lockTime) + err := wallet.dataStore.UTXOs().Put(&output.ProgramHash, utxo) + if err != nil { + return false, err + } + hits++ } } - if totalOutputValue > 0 { - return nil, errors.New("[Wallet], Available token is not enough") - } - addr, err := wallet.GetAddress(spender) - if err != nil { - return nil, errors.New("[Wallet], Get spenders redeem script failed") + // Put spent UTXOs to STXOs + for _, input := range tx.Inputs { + // Try to move UTXO to STXO, if a UTXO in database was spent, it will be moved to STXO + err := wallet.dataStore.STXOs().FromUTXO(&input.Previous, &txId, height) + if err == nil { + hits++ + } } - return wallet.newTransaction(addr.Script(), txInputs, txOutputs), nil -} + // If no hits, no need to save transaction + if hits == 0 { + return true, nil + } -func (wallet *WalletImpl) Sign(password []byte, txn *core.Transaction) (*core.Transaction, error) { - // Verify password - err := wallet.VerifyPassword(password) + // Save transaction + err := wallet.dataStore.Txs().Put(db.NewTx(*tx, height)) if err != nil { - return nil, err + return false, err } - // Get sign type - signType, err := crypto.GetScriptType(txn.Programs[0].Code) - if err != nil { - return nil, err - } - // Look up transaction type - if signType == common.STANDARD { - // Sign single transaction - txn, err = wallet.signStandardTransaction(txn) - if err != nil { - return nil, err - } + wallet.txIds.Add(txId, height) - } else if signType == common.MULTISIG { + return false, nil +} + +func (wallet *SPVWallet) OnBlockCommitted(block *msg.MerkleBlock, txs []*core.Transaction) { + wallet.dataStore.Chain().PutHeight(block.Header.(*core.Header).Height) - // Sign multi sign transaction - txn, err = wallet.signMultiSigTransaction(txn) + // Check unconfirmed transaction timeout + if wallet.IsCurrent() { + // Get all unconfirmed transactions + txs, err := wallet.dataStore.Txs().GetAllFrom(0) if err != nil { - return nil, err + log.Debugf("Get unconfirmed transactions failed, error %s", err.Error()) + return + } + now := time.Now() + for _, tx := range txs { + if now.After(tx.Timestamp.Add(MaxUnconfirmedTime)) { + err = wallet.dataStore.RollbackTx(&tx.TxId) + if err != nil { + log.Errorf("Rollback timeout transaction %s failed, error %s", tx.TxId.String(), err.Error()) + } + } } } - - return txn, nil } -func (wallet *WalletImpl) signStandardTransaction(txn *core.Transaction) (*core.Transaction, error) { - code := txn.Programs[0].Code - // Get signer - programHash, err := crypto.GetSigner(code) - // Check if current user is a valid signer - account := wallet.Keystore.GetAccountByProgramHash(programHash) - if account == nil { - return nil, errors.New("[Wallet], Invalid signer") - } - // Sign transaction - buf := new(bytes.Buffer) - txn.SerializeUnsigned(buf) - signature, err := account.Sign(buf.Bytes()) - if err != nil { - return nil, err - } - // Add signature - buf = new(bytes.Buffer) - buf.WriteByte(byte(len(signature))) - buf.Write(signature) - // Set program - var program = &core.Program{code, buf.Bytes()} - txn.Programs = []*core.Program{program} - - return txn, nil +// Rollback chain data on the given height +func (wallet *SPVWallet) OnRollback(height uint32) error { + return wallet.dataStore.Rollback(height) } -func (wallet *WalletImpl) signMultiSigTransaction(txn *core.Transaction) (*core.Transaction, error) { - code := txn.Programs[0].Code - param := txn.Programs[0].Parameter - // Check if current user is a valid signer - var signerIndex = -1 - programHashes, err := crypto.GetSigners(code) - if err != nil { - return nil, err - } - var account *sdk.Account - for i, programHash := range programHashes { - account = wallet.Keystore.GetAccountByProgramHash(programHash) - if account != nil { - signerIndex = i - break - } - } - if signerIndex == -1 { - return nil, errors.New("[Wallet], Invalid multi sign signer") - } - // Sign transaction - buf := new(bytes.Buffer) - txn.SerializeUnsigned(buf) - signedTx, err := account.Sign(buf.Bytes()) - if err != nil { - return nil, err - } - // Append signature - txn.Programs[0].Parameter, err = crypto.AppendSignature(signerIndex, signedTx, buf.Bytes(), code, param) - if err != nil { - return nil, err - } +func ToUTXO(txId common.Uint256, height uint32, index int, value common.Fixed64, lockTime uint32) *db.UTXO { + utxo := new(db.UTXO) + utxo.Op = *core.NewOutPoint(txId, uint16(index)) + utxo.Value = value + utxo.LockTime = lockTime + utxo.AtHeight = height + return utxo +} - return txn, nil +func (wallet *SPVWallet) NotifyNewAddress(hash []byte) { + // Reload address filter to include new address + wallet.loadAddrFilter() + // Broadcast filterload message to connected peers + wallet.UpdateFilter() } -func (wallet *WalletImpl) SendTransaction(txn *core.Transaction) error { - // Send transaction through P2P network - return rpc.GetClient().SendTransaction(txn) +func (wallet *SPVWallet) getAddrFilter() *sdk.AddrFilter { + if wallet.filter == nil { + wallet.loadAddrFilter() + } + return wallet.filter } -func getSystemAssetId() common.Uint256 { - systemToken := &core.Transaction{ - TxType: core.RegisterAsset, - PayloadVersion: 0, - Payload: &core.PayloadRegisterAsset{ - Asset: core.Asset{ - Name: "ELA", - Precision: 0x08, - AssetType: 0x00, - }, - Amount: 0 * 100000000, - Controller: common.Uint168{}, - }, - Attributes: []*core.Attribute{}, - Inputs: []*core.Input{}, - Outputs: []*core.Output{}, - Programs: []*core.Program{}, +func (wallet *SPVWallet) loadAddrFilter() *sdk.AddrFilter { + addrs, _ := wallet.dataStore.Addrs().GetAll() + wallet.filter = sdk.NewAddrFilter(nil) + for _, addr := range addrs { + wallet.filter.AddAddr(addr.Hash()) } - return systemToken.Hash() + return wallet.filter } -func (wallet *WalletImpl) removeLockedUTXOs(utxos []*UTXO) []*UTXO { - var availableUTXOs []*UTXO - var currentHeight = wallet.ChainHeight() - for _, utxo := range utxos { - if utxo.AtHeight == 0 { // remove unconfirmed UTOXs +// checkDoubleSpends takes a transaction and compares it with +// all transactions in the db. It returns a slice of all txIds in the db +// which are double spent by the received tx. +func (wallet *SPVWallet) checkDoubleSpends(tx *core.Transaction) ([]*common.Uint256, error) { + var dubs []*common.Uint256 + txId := tx.Hash() + txs, err := wallet.dataStore.Txs().GetAll() + if err != nil { + return nil, err + } + for _, compTx := range txs { + // Skip coinbase transaction + if compTx.Data.IsCoinBaseTx() { + continue + } + // Skip duplicate transaction + compTxId := compTx.Data.Hash() + if compTxId.IsEqual(txId) { continue } - if utxo.LockTime > 0 { - if utxo.LockTime > currentHeight { - continue + for _, txIn := range tx.Inputs { + for _, compIn := range compTx.Data.Inputs { + if txIn.Previous.IsEqual(compIn.Previous) { + // Found double spend + dubs = append(dubs, &compTxId) + break // back to txIn loop + } } - utxo.LockTime = math.MaxUint32 - 1 } - availableUTXOs = append(availableUTXOs, utxo) - } - return availableUTXOs -} - -func InputFromUTXO(utxo *UTXO) *core.Input { - input := new(core.Input) - input.Previous.TxID = utxo.Op.TxID - input.Previous.Index = utxo.Op.Index - input.Sequence = utxo.LockTime - return input -} - -func (wallet *WalletImpl) newTransaction(redeemScript []byte, inputs []*core.Input, outputs []*core.Output) *core.Transaction { - // Create payload - txPayload := &core.PayloadTransferAsset{} - // Create attributes - txAttr := core.NewAttribute(core.Nonce, []byte(strconv.FormatInt(rand.Int63(), 10))) - attributes := make([]*core.Attribute, 0) - attributes = append(attributes, &txAttr) - // Create program - var program = &core.Program{redeemScript, nil} - // Create transaction - return &core.Transaction{ - TxType: core.TransferAsset, - Payload: txPayload, - Attributes: attributes, - Inputs: inputs, - Outputs: outputs, - Programs: []*core.Program{program}, - LockTime: wallet.ChainHeight(), } + return dubs, nil } diff --git a/store/headerstore.go b/store/headerstore.go deleted file mode 100644 index d270e74..0000000 --- a/store/headerstore.go +++ /dev/null @@ -1,23 +0,0 @@ -package store - -import "github.com/elastos/Elastos.ELA.Utility/common" - -type HeaderStore interface { - // Save a header to database - PutHeader(header *StoreHeader, newTip bool) error - - // Get previous block of the given header - GetPrevious(header *StoreHeader) (*StoreHeader, error) - - // Get full header with it's hash - GetHeader(hash *common.Uint256) (*StoreHeader, error) - - // Get the header on chain tip - GetBestHeader() (*StoreHeader, error) - - // Reset header store - Reset() error - - // Close header store - Close() -} diff --git a/store/storeheader.go b/util/header.go similarity index 67% rename from store/storeheader.go rename to util/header.go index 4dc573d..ff8d7d5 100644 --- a/store/storeheader.go +++ b/util/header.go @@ -1,4 +1,4 @@ -package store +package database import ( "bytes" @@ -7,12 +7,17 @@ import ( "github.com/elastos/Elastos.ELA/core" ) -type StoreHeader struct { - core.Header +// Header is a data structure stored in database. +type Header struct { + // The origin header of the block + *core.Header + + // The total work from the genesis block to this + // current block TotalWork *big.Int } -func (sh *StoreHeader) Serialize() ([]byte, error) { +func (sh *Header) Serialize() ([]byte, error) { buf := new(bytes.Buffer) err := sh.Header.Serialize(buf) if err != nil { @@ -26,7 +31,7 @@ func (sh *StoreHeader) Serialize() ([]byte, error) { return buf.Bytes(), nil } -func (sh *StoreHeader) Deserialize(b []byte) error { +func (sh *Header) Deserialize(b []byte) error { r := bytes.NewReader(b) err := sh.Header.Deserialize(r) if err != nil { From 52ed37c1a83d4940d843d2b4b98a1491debbc197 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 8 Sep 2018 13:30:58 +0800 Subject: [PATCH 05/73] initial database package implement --- database/chainstore.go | 34 +++++++++++++++ database/db.go | 10 +++++ database/defaultdb.go | 92 +++++++++++++++++++++++++++++++++++++++++ database/headers.go | 9 ++-- database/headersonly.go | 45 ++++++++++++++++++++ database/txsdb.go | 49 ++++++++++++++++++++++ 6 files changed, 233 insertions(+), 6 deletions(-) create mode 100644 database/chainstore.go create mode 100644 database/db.go create mode 100644 database/defaultdb.go create mode 100644 database/headersonly.go create mode 100644 database/txsdb.go diff --git a/database/chainstore.go b/database/chainstore.go new file mode 100644 index 0000000..c26abc8 --- /dev/null +++ b/database/chainstore.go @@ -0,0 +1,34 @@ +package database + +import ( + "github.com/elastos/Elastos.ELA.SPV/util" +) + +type ChainStore interface { + // Extend from DB interface + DB + + // Headers returns the headers database that stored + // all blockchain headers. + Headers() Headers + + // StoreBlock save a block into database, returns how many + // false positive transactions are and error. + StoreBlock(block *util.Block, newTip bool) (fps uint32, err error) + + // StoreTx save a transaction into database, and return + // if it is a false positive and error. + StoreTx(tx *util.Tx) (bool, error) + + // Rollback delete all transactions after the reorg point, + // it is used when blockchain reorganized. + Rollback(reorg *util.Header) error +} + +func NewHeadersOnlyChainDB(db Headers) ChainStore { + return &headersOnlyChainDB{db: db} +} + +func NewDefaultChainDB(h Headers, t TxsDB) ChainStore { + return &defaultChainDB{h: h, t: t} +} diff --git a/database/db.go b/database/db.go new file mode 100644 index 0000000..f1e492c --- /dev/null +++ b/database/db.go @@ -0,0 +1,10 @@ +package database + +// DB is the common interface to all database implementations. +type DB interface { + // Clear delete all data in database. + Clear() error + + // Close database. + Close() error +} diff --git a/database/defaultdb.go b/database/defaultdb.go new file mode 100644 index 0000000..a3410f5 --- /dev/null +++ b/database/defaultdb.go @@ -0,0 +1,92 @@ +package database + +import "github.com/elastos/Elastos.ELA.SPV/util" + +type defaultChainDB struct { + h Headers + t TxsDB +} + +// Headers returns the headers database that stored +// all blockchain headers. +func (d *defaultChainDB) Headers() Headers { + return d.h +} + +// StoreBlock save a block into database, returns how many +// false positive transactions are and error. +func (d *defaultChainDB) StoreBlock(block *util.Block, newTip bool) (fps uint32, err error) { + err = d.h.Put(block.Header, newTip) + if err != nil { + return 0, err + } + + // We are on a fork chain, do not commit transactions. + if !newTip { + return 0, nil + } + + batch := d.t.Batch() + for _, tx := range block.Transactions { + fp, err := batch.AddTx(tx) + if err != nil { + return 0, batch.Rollback() + } + if fp { + fps++ + } + } + + return fps, batch.Commit() +} + +// StoreTx save a transaction into database, and return +// if it is a false positive and error. +func (d *defaultChainDB) StoreTx(tx *util.Tx) (bool, error) { + return d.t.CommitTx(tx) +} + +// RollbackTo delete all transactions after the reorg point, +// it is used when blockchain reorganized. +func (d *defaultChainDB) Rollback(reorg *util.Header) error { + // Get current chain tip + best, err := d.h.GetBest() + if err != nil { + return err + } + + batch := d.t.Batch() + for current := best.Height; current > reorg.Height; current-- { + if err := batch.DelTxs(current); err != nil { + return batch.Rollback() + } + } + + if err := batch.Commit(); err != nil { + return err + } + + return d.h.Put(reorg, true) +} + +// Clear delete all data in database. +func (d *defaultChainDB) Clear() error { + if err := d.h.Clear(); err != nil { + return err + } + if err := d.t.Clear(); err != nil { + return err + } + return nil +} + +// Close database. +func (d *defaultChainDB) Close() error { + if err := d.h.Close(); err != nil { + return err + } + if err := d.t.Close(); err != nil { + return err + } + return nil +} diff --git a/database/headers.go b/database/headers.go index c02bbd4..e2b44ea 100644 --- a/database/headers.go +++ b/database/headers.go @@ -11,17 +11,14 @@ type Headers interface { DB // Save a header to database - PutHeader(header *util.Header, newTip bool) error + Put(header *util.Header, newTip bool) error // Get previous block of the given header GetPrevious(header *util.Header) (*util.Header, error) // Get full header with it's hash - GetHeader(hash *common.Uint256) (*util.Header, error) + Get(hash *common.Uint256) (*util.Header, error) // Get the header on chain tip - GetBestHeader() (*util.Header, error) - - // DelHeader delete a header save in database by it's hash. - DelHeader(hash *common.Uint256) error + GetBest() (*util.Header, error) } diff --git a/database/headersonly.go b/database/headersonly.go new file mode 100644 index 0000000..cd9068d --- /dev/null +++ b/database/headersonly.go @@ -0,0 +1,45 @@ +package database + +import ( + "github.com/elastos/Elastos.ELA.SPV/util" +) + +type headersOnlyChainDB struct { + db Headers +} + +// Headers returns the headers database that stored +// all blockchain headers. +func (h *headersOnlyChainDB) Headers() Headers { + return h.db +} + +// StoreBlock save a block into database, returns how many +// false positive transactions are and error. +func (h *headersOnlyChainDB) StoreBlock(block *util.Block, newTip bool) (fps uint32, err error) { + return fps, h.db.Put(block.Header, newTip) +} + +// StoreTx save a transaction into database, and return +// if it is a false positive and error. +func (h *headersOnlyChainDB) StoreTx(tx *util.Tx) (bool, error) { + return false, nil +} + +// RollbackTo delete all transactions after the reorg point, +// it is used when blockchain reorganized. +func (h *headersOnlyChainDB) Rollback(reorg *util.Header) error { + // Just do nothing. Headers never removed from database, + // only transactions need to be rollback. + return nil +} + +// Clear delete all data in database. +func (h *headersOnlyChainDB) Clear() error { + return h.db.Clear() +} + +// Close database. +func (h *headersOnlyChainDB) Close() error { + return h.db.Close() +} diff --git a/database/txsdb.go b/database/txsdb.go new file mode 100644 index 0000000..21f79ad --- /dev/null +++ b/database/txsdb.go @@ -0,0 +1,49 @@ +package database + +import ( + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" +) + +type TxsDB interface { + // Extend from DB interface + DB + + // Batch returns a TxBatch instance for transactions batch + // commit, this can get better performance when commit a bunch + // of transactions within a block. + Batch() TxBatch + + // CommitTx save a transaction into database, and return + // if it is a false positive and error. + CommitTx(tx *util.Tx) (bool, error) + + // HaveTx returns if the transaction already saved in database + // by it's id. + HaveTx(txId *common.Uint256) (bool, error) + + // GetTxs returns all transactions within the given height. + GetTxs(height uint32) ([]*util.Tx, error) + + // RemoveTxs delete all transactions on the given height. Return + // how many transactions are deleted from database. + RemoveTxs(height uint32) (int, error) +} + +type TxBatch interface { + // AddTx add a store transaction operation into batch, and return + // if it is a false positive and error. + AddTx(tx *util.Tx) (bool, error) + + // DelTx add a delete transaction operation into batch. + DelTx(txId *common.Uint256) error + + // DelTxs add a delete transactions on given height operation. + DelTxs(height uint32) error + + // Rollback cancel all operations in current batch. + Rollback() error + + // Commit the added transactions into database. + Commit() error +} From 43fbe19ff59aade16111f583e446a338a7ea1876 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 8 Sep 2018 13:34:14 +0800 Subject: [PATCH 06/73] initial peer package implement --- peer/log.go | 28 +++ peer/peer.go | 515 ++++++++++++++++++++++++++++++++++----------------- 2 files changed, 371 insertions(+), 172 deletions(-) create mode 100644 peer/log.go diff --git a/peer/log.go b/peer/log.go new file mode 100644 index 0000000..eddf96c --- /dev/null +++ b/peer/log.go @@ -0,0 +1,28 @@ +package peer + +import ( + "github.com/elastos/Elastos.ELA.Utility/elalog" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log elalog.Logger + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = elalog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using elalog. +func UseLogger(logger elalog.Logger) { + log = logger +} diff --git a/peer/peer.go b/peer/peer.go index 569c1a8..1c9de2f 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -1,15 +1,17 @@ -package sdk +package peer import ( + "container/list" + "sync" "time" - "github.com/elastos/Elastos.ELA.SPV/log" - "github.com/elastos/Elastos.ELA.SPV/net" + "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA.Utility/p2p/rw" + "github.com/elastos/Elastos.ELA.Utility/p2p/peer" + "github.com/elastos/Elastos.ELA/bloom" "github.com/elastos/Elastos.ELA/core" ) @@ -23,193 +25,123 @@ const ( // stalling. The deadlines are adjusted for callback running times and // only checked on each stall tick interval. stallResponseTimeout = 15 * time.Second -) - -func (p *SPVPeer) EnqueueBlock(header *msg.MerkleBlock, txIds []*common.Uint256) *block { - hash := header.Header.(*core.Header).Hash() - - block := newBlock(header) - - // No transactions to download, just finish it - if len(txIds) == 0 { - return block - } - - // Download transactions of this block - getData := msg.NewGetData() - for _, txId := range txIds { - block.txQueue[*txId] = struct{}{} - getData.AddInvVect(msg.NewInvVect(msg.InvTypeTx, txId)) - } - // Stall message - p.stallControl <- getData - - p.blocksQueue[hash] = block - - return nil -} - -func (p *SPVPeer) EnqueueTx(txId common.Uint256) { - p.txsQueue[txId] = struct{}{} -} - -func (p *SPVPeer) DequeueTx(tx *core.Transaction) (download interface{}, finished bool) { - // Get tx id - txId := tx.Hash() - - // Check downloading block first - if blockId, ok := p.blockTxs[txId]; ok { - // Dequeue block tx - block := p.blocksQueue[blockId] - block.txs = append(block.txs, tx) - delete(block.txQueue, txId) - - return block, len(block.txQueue) == 0 - } - - // Dequeue from downloading tx - _, ok := p.txsQueue[txId] - if !ok { - return tx, false - } - delete(p.txsQueue, txId) - - return tx, true -} - -type block struct { - *msg.MerkleBlock - txQueue map[common.Uint256]struct{} - txs []*core.Transaction -} - -func newBlock(header *msg.MerkleBlock) *block { - return &block{ - MerkleBlock: header, - txQueue: make(map[common.Uint256]struct{}), - } -} -type SPVPeerConfig struct { - // LocalHeight is invoked when peer queue a ping or pong message - LocalHeight func() uint32 + // outputBufferSize is the number of elements the output channels use. + outputBufferSize = 50 +) +// Config is the struct to hold configuration options useful to Peer. +type Config struct { // After send a blocks request message, this inventory message // will return with a bunch of block hashes, then you can use them // to request all the blocks by send data requests. - OnInventory func(*SPVPeer, *msg.Inventory) error - - // After sent a data request with invType BLOCK, a merkleblock message will return through this method. - // To make this work, you must register a filterload message to the connected peer first, - // then this client will be known as a SPV client. To create a bloom filter and get the - // filterload message, you will use the method in SDK bloom sdk.NewBloomFilter() - // merkleblock includes a block header, transaction hashes in merkle proof format. - // Which transaction hashes will be in the merkleblock is depends on the addresses and outpoints - // you've added into the bloom filter before you send a filterload message with this bloom filter. - // You will use these transaction hashes to request transactions by sending data request message - // with invType TRANSACTION - OnMerkleBlock func(*SPVPeer, *msg.MerkleBlock) error + OnInv func(*Peer, *msg.Inv) + + // This method will be invoked when a merkleblock and transactions + // within it has been downloaded. + OnBlock func(*Peer, *util.Block) // After sent a data request with invType TRANSACTION, a txn message will return through this method. // these transactions are matched to the bloom filter you have sent with the filterload message. - OnTx func(*SPVPeer, *msg.Tx) error + OnTx func(*Peer, *core.Transaction) // If the BLOCK or TRANSACTION requested by the data request message can not be found, // notfound message with requested data hash will return through this method. - OnNotFound func(*SPVPeer, *msg.NotFound) error + OnNotFound func(*Peer, *msg.NotFound) // If the submitted transaction was rejected, this message will return. - OnReject func(*SPVPeer, *msg.Reject) error + OnReject func(*Peer, *msg.Reject) } -type SPVPeer struct { - *net.Peer +// outMsg is used to house a message to be sent along with a channel to signal +// when the message has been sent (or won't be sent due to things such as +// shutdown) +type outMsg struct { + msg p2p.Message + doneChan chan<- struct{} +} + +type Peer struct { + *peer.Peer + cfg Config - blockQueue chan common.Uint256 - blocksQueue map[common.Uint256]*block - blockTxs map[common.Uint256]common.Uint256 - txsQueue map[common.Uint256]struct{} - receivedTxs int - fPositives int + prevGetBlocksMtx sync.Mutex + prevGetBlocksBegin *common.Uint256 + prevGetBlocksStop *common.Uint256 stallControl chan p2p.Message + blockQueue chan p2p.Message + outputQueue chan outMsg + queueQuit chan struct{} } -func NewSPVPeer(peer *net.Peer, config SPVPeerConfig) *SPVPeer { - spvPeer := &SPVPeer{ +func NewPeer(peer *peer.Peer, cfg *Config) *Peer { + p := Peer{ Peer: peer, - blockQueue: make(chan common.Uint256, p2p.MaxBlocksPerMsg), - blocksQueue: make(map[common.Uint256]*block), - blockTxs: make(map[common.Uint256]common.Uint256), - txsQueue: make(map[common.Uint256]struct{}), + cfg: *cfg, stallControl: make(chan p2p.Message, 1), + blockQueue: make(chan p2p.Message, 1), + outputQueue: make(chan outMsg, outputBufferSize), + queueQuit: make(chan struct{}), } + peer.AddMessageFunc(p.handleMessage) + + go p.stallHandler() + go p.blockHandler() + + go func() { + // We have waited on queueQuit and thus we can be sure + // that we will not miss anything sent on sendQueue. + <-p.queueQuit + p.CleanupSendQueue() + }() + return &p +} - msgConfig := rw.Config{ - ProtocolVersion: p2p.EIP001Version, - MakeTx: func() *msg.Tx { return msg.NewTx(new(core.Transaction)) }, - MakeBlock: func() *msg.Block { return msg.NewBlock(new(core.Block)) }, - MakeMerkleBlock: func() *msg.MerkleBlock { return msg.NewMerkleBlock(new(core.Header)) }, - } - - spvPeer.SetMessageConfig(msgConfig) - - peerConfig := net.PeerConfig{ - PingNonce: config.LocalHeight, - - PongNonce: config.LocalHeight, - - OnPing: func(peer *net.Peer, ping *msg.Ping) { - peer.SetHeight(ping.Nonce) - }, - - OnPong: func(peer *net.Peer, pong *msg.Pong) { - peer.SetHeight(pong.Nonce) - }, +func (p *Peer) sendMessage(msg outMsg) { + p.stallControl <- msg.msg + p.SendMessage(msg.msg, msg.doneChan) +} - HandleMessage: func(peer *net.Peer, message p2p.Message) { - // Notify stall control - spvPeer.stallControl <- message +func (p *Peer) handleMessage(peer *peer.Peer, message p2p.Message) { + // Notify stall control + p.stallControl <- message - switch m := message.(type) { - case *msg.Inventory: - config.OnInventory(spvPeer, m) + switch m := message.(type) { + case *msg.Inv: + p.cfg.OnInv(p, m) - case *msg.MerkleBlock: - config.OnMerkleBlock(spvPeer, m) + case *msg.MerkleBlock: + p.blockQueue <- m - case *msg.Tx: - config.OnTx(spvPeer, m) + case *msg.Tx: + p.blockQueue <- m - case *msg.NotFound: - config.OnNotFound(spvPeer, m) + case *msg.NotFound: + p.cfg.OnNotFound(p, m) - case *msg.Reject: - config.OnReject(spvPeer, m) - } - }, + case *msg.Reject: + p.cfg.OnReject(p, m) } +} - spvPeer.SetPeerConfig(peerConfig) - - go spvPeer.stallHandler() +func (p *Peer) stallHandler() { + // lastActive tracks the last active sync message. + var lastActive time.Time - return spvPeer -} + // pendingResponses tracks the expected responses. + pendingResponses := make(map[string]struct{}) -func (p *SPVPeer) stallHandler() { // stallTicker is used to periodically check pending responses that have // exceeded the expected deadline and disconnect the peer due to stalling. stallTicker := time.NewTicker(stallTickInterval) defer stallTicker.Stop() - // pendingResponses tracks the expected responses. - pendingResponses := make(map[string]struct{}) - - // lastActive tracks the last active sync message. - var lastActive time.Time - - for p.Connected() { + // ioStopped is used to detect when both the input and output handler + // goroutines are done. + var ioStopped bool +out: + for { select { case ctrMsg := <-p.stallControl: // update last active time @@ -220,7 +152,7 @@ func (p *SPVPeer) stallHandler() { // Add expected response pendingResponses[p2p.CmdInv] = struct{}{} - case *msg.Inventory: + case *msg.Inv: // Remove inventory from expected response map. delete(pendingResponses, p2p.CmdInv) @@ -236,7 +168,7 @@ func (p *SPVPeer) stallHandler() { case *msg.Tx: // Remove received transaction from expected response map. - delete(pendingResponses, message.Transaction.(*core.Transaction).Hash().String()) + delete(pendingResponses, message.Serializable.(*core.Transaction).Hash().String()) case *msg.NotFound: // NotFound should not received from sync peer @@ -257,6 +189,22 @@ func (p *SPVPeer) stallHandler() { log.Debugf("peer %v appears to be stalled or misbehaving, response timeout -- disconnecting", p) p.Disconnect() + + case <-p.InQuit(): + // The stall handler can exit once both the input and + // output handler goroutines are done. + if ioStopped { + break out + } + ioStopped = true + + case <-p.OutQuit(): + // The stall handler can exit once both the input and + // output handler goroutines are done. + if ioStopped { + break out + } + ioStopped = true } } @@ -273,30 +221,253 @@ cleanup: log.Tracef("Peer stall handler done for %v", p) } -// Add message to output queue and wait until message sent -func (p *SPVPeer) SendMessage(message p2p.Message) { - doneChan := make(chan struct{}) - p.queueMessage(message, doneChan) - <-doneChan -} +// blockHandler handles the downloading of merkleblock and the transactions +// within it. We handle a merkleblock and it's transactions as one block. +// We do not notify this new block until the downloading has been completed. +func (p *Peer) blockHandler() { + // Data caches for the downloading block. + var header *util.Header + var pendingTxs map[common.Uint256]struct{} + var txs []*util.Tx + + // NotifyOnBlock message and clear cached data. + notifyBlock := func() { + // Notify OnBlock. + p.cfg.OnBlock(p, &util.Block{ + Header: header, + Transactions: txs, + }) + + // Clear cached data. + header = nil + pendingTxs = nil + txs = nil + } -// Add a message into output queue -func (p *SPVPeer) QueueMessage(message p2p.Message) { - p.queueMessage(message, nil) +out: + for { + select { + case bmsg := <-p.blockQueue: + switch m := bmsg.(type) { + case *msg.MerkleBlock: + // If header is not nil, the previous block download was not + // finished, that means the peer is misbehaving, disconnect it. + if header != nil { + log.Debugf("peer %v send us new block before previous"+ + " block download finished -- disconnecting", p) + p.Disconnect() + continue + } + + // Check block + txIds, err := bloom.CheckMerkleBlock(*m) + if err != nil { + log.Debugf("peer %v send us invalid merkleblock -- disconnecting", p) + p.Disconnect() + continue + } + + // No transaction included in this block, so just notify block + // downloading completed. + if len(txIds) == 0 { + notifyBlock() + continue + } + + // Set current downloading block + header = &util.Header{ + Header: m.Header.(*core.Header), + NumTxs: m.Transactions, + Hashes: m.Hashes, + Flags: m.Flags, + } + + // Save pending transactions to cache. + pendingTxs = make(map[common.Uint256]struct{}) + for _, txId := range txIds { + pendingTxs[*txId] = struct{}{} + } + + // Initiate transactions cache. + txs = make([]*util.Tx, 0, len(pendingTxs)) + + case *msg.Tx: + // Not in block downloading mode, just notify new transaction. + tx := m.Serializable.(*core.Transaction) + if header == nil { + p.cfg.OnTx(p, tx) + continue + } + + txId := tx.Hash() + + // When downloading block, received transactions can only by + // those within the block. + if _, ok := pendingTxs[txId]; !ok { + log.Debugf("peer %v send us invalid transaction -- disconnecting", p) + p.Disconnect() + continue + } + + // Save downloaded transaction to cache. + txs = append(txs, &util.Tx{ + Transaction: tx, + Height: header.Height, + }) + + // Remove transaction from pending list. + delete(pendingTxs, txId) + + // Block download completed, notify OnBlock. + if len(pendingTxs) == 0 { + notifyBlock() + } + } + + case <-p.Quit(): + break out + } + } + + // Drain any wait channels before going away so there is nothing left + // waiting on this goroutine. +cleanup: + for { + select { + case <-p.blockQueue: + default: + break cleanup + } + } + log.Tracef("Peer block handler done for %v", p) } -func (p *SPVPeer) queueMessage(message p2p.Message, doneChan chan struct{}) { - switch message.(type) { - case *msg.GetBlocks, *msg.GetData: - p.stallControl <- message +// queueHandler handles the queuing of outgoing data for the peer. This runs as +// a muxer for various sources of input so we can ensure that server and peer +// handlers will not block on us sending a message. That data is then passed on +// to outHandler to be actually written. +func (p *Peer) queueHandler() { + pendingMsgs := list.New() + + // We keep the waiting flag so that we know if we have a message queued + // to the outHandler or not. We could use the presence of a head of + // the list for this but then we have rather racy concerns about whether + // it has gotten it at cleanup time - and thus who sends on the + // message's done channel. To avoid such confusion we keep a different + // flag and pendingMsgs only contains messages that we have not yet + // passed to outHandler. + waiting := false + + // To avoid duplication below. + queuePacket := func(msg outMsg, list *list.List, waiting bool) bool { + if !waiting { + p.sendMessage(msg) + } else { + list.PushBack(msg) + } + // we are always waiting now. + return true + } +out: + for { + select { + case msg := <-p.outputQueue: + waiting = queuePacket(msg, pendingMsgs, waiting) + + // This channel is notified when a message has been sent across + // the network socket. + case <-p.SendDoneQueue(): + // No longer waiting if there are no more messages + // in the pending messages queue. + next := pendingMsgs.Front() + if next == nil { + waiting = false + continue + } + + // Notify the outHandler about the next item to + // asynchronously send. + val := pendingMsgs.Remove(next) + p.sendMessage(val.(outMsg)) + + case <-p.Quit(): + break out + } + } + + // Drain any wait channels before we go away so we don't leave something + // waiting for us. + for e := pendingMsgs.Front(); e != nil; e = pendingMsgs.Front() { + val := pendingMsgs.Remove(e) + msg := val.(outMsg) + if msg.doneChan != nil { + msg.doneChan <- struct{}{} + } } - p.Peer.QueueMessage(message, doneChan) +cleanup: + for { + select { + case msg := <-p.outputQueue: + if msg.doneChan != nil { + msg.doneChan <- struct{}{} + } + default: + break cleanup + } + } + close(p.queueQuit) + log.Tracef("Peer queue handler done for %s", p) } -func (p *SPVPeer) GetFalsePositiveRate() float32 { - return float32(p.fPositives) / float32(p.receivedTxs) +// PushGetBlocksMsg sends a getblocks message for the provided block locator +// and stop hash. It will ignore back-to-back duplicate requests. +// +// This function is safe for concurrent access. +func (p *Peer) PushGetBlocksMsg(locator []*common.Uint256, stopHash *common.Uint256) error { + // Extract the begin hash from the block locator, if one was specified, + // to use for filtering duplicate getblocks requests. + var beginHash *common.Uint256 + if len(locator) > 0 { + beginHash = locator[0] + } + + // Filter duplicate getblocks requests. + p.prevGetBlocksMtx.Lock() + isDuplicate := p.prevGetBlocksStop != nil && p.prevGetBlocksBegin != nil && + beginHash != nil && stopHash.IsEqual(*p.prevGetBlocksStop) && + beginHash.IsEqual(*p.prevGetBlocksBegin) + p.prevGetBlocksMtx.Unlock() + + if isDuplicate { + log.Tracef("Filtering duplicate [getblocks] with begin "+ + "hash %v, stop hash %v", beginHash, stopHash) + return nil + } + + // Construct the getblocks request and queue it to be sent. + msg := msg.NewGetBlocks(locator, *stopHash) + p.QueueMessage(msg, nil) + + // Update the previous getblocks request information for filtering + // duplicates. + p.prevGetBlocksMtx.Lock() + p.prevGetBlocksBegin = beginHash + p.prevGetBlocksStop = stopHash + p.prevGetBlocksMtx.Unlock() + return nil } -func (p *SPVPeer) ResetFalsePositives() { - p.fPositives, p.receivedTxs = 0, 0 +func (p *Peer) QueueMessage(msg p2p.Message, doneChan chan<- struct{}) { + // Avoid risk of deadlock if goroutine already exited. The goroutine + // we will be sending to hangs around until it knows for a fact that + // it is marked as disconnected and *then* it drains the channels. + if !p.Connected() { + if doneChan != nil { + go func() { + doneChan <- struct{}{} + }() + } + return + } + p.outputQueue <- outMsg{msg: msg, doneChan: doneChan} } From ae7b2c07c1ff88f0db76c48575f5b464fa40e274 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 8 Sep 2018 13:34:31 +0800 Subject: [PATCH 07/73] initial sync package implement --- sync/config.go | 31 ++ sync/log.go | 28 ++ sync/manager.go | 782 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 841 insertions(+) create mode 100644 sync/config.go create mode 100644 sync/log.go create mode 100644 sync/manager.go diff --git a/sync/config.go b/sync/config.go new file mode 100644 index 0000000..b53e9b2 --- /dev/null +++ b/sync/config.go @@ -0,0 +1,31 @@ +package sync + +import ( + "github.com/elastos/Elastos.ELA.SPV/blockchain" + "github.com/elastos/Elastos.ELA/bloom" +) + +const ( + DefaultMinPeersForSync = 3 + DefaultMaxPeers = 125 +) + +// Config is a configuration struct used to initialize a new SyncManager. +type Config struct { + Chain *blockchain.BlockChain + + MinPeersForSync int + MaxPeers int + + UpdateFilter func() *bloom.Filter +} + +func NewDefaultConfig(chain *blockchain.BlockChain, + updateFilter func() *bloom.Filter) *Config { + return &Config{ + Chain: chain, + MinPeersForSync: DefaultMinPeersForSync, + MaxPeers: DefaultMaxPeers, + UpdateFilter: updateFilter, + } +} diff --git a/sync/log.go b/sync/log.go new file mode 100644 index 0000000..794090f --- /dev/null +++ b/sync/log.go @@ -0,0 +1,28 @@ +package sync + +import ( + "github.com/elastos/Elastos.ELA.Utility/elalog" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log elalog.Logger + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = elalog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using elalog. +func UseLogger(logger elalog.Logger) { + log = logger +} diff --git a/sync/manager.go b/sync/manager.go new file mode 100644 index 0000000..e8c6690 --- /dev/null +++ b/sync/manager.go @@ -0,0 +1,782 @@ +package sync + +import ( + "sync" + "sync/atomic" + + "github.com/elastos/Elastos.ELA.SPV/blockchain" + "github.com/elastos/Elastos.ELA.SPV/peer" + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/core" +) + +const ( + // maxBadBlockRate is the maximum bad blocks rate of received blocks. + maxBadBlockRate float64 = 0.001 + + // maxFalsePositiveRate is the maximum false positive rate of received + // transactions. + maxFalsePositiveRate float64 = 0.001 + + // maxRequestedBlocks is the maximum number of requested block + // hashes to store in memory. + maxRequestedBlocks = msg.MaxInvPerMsg + + // maxRequestedTxns is the maximum number of requested transactions + // hashes to store in memory. + maxRequestedTxns = msg.MaxInvPerMsg +) + +// zeroHash is the zero value hash (all zeros). It is defined as a convenience. +var zeroHash common.Uint256 + +// newPeerMsg signifies a newly connected peer to the block handler. +type newPeerMsg struct { + peer *peer.Peer +} + +// donePeerMsg signifies a newly disconnected peer to the block handler. +type donePeerMsg struct { + peer *peer.Peer +} + +// invMsg packages a bitcoin inv message and the peer it came from together +// so the block handler has access to that information. +type invMsg struct { + inv *msg.Inv + peer *peer.Peer +} + +// blockMsg packages a block message and the peer it came from +// together so the block handler has access to that information. +type blockMsg struct { + block *util.Block + peer *peer.Peer + reply chan struct{} +} + +// txMsg packages a bitcoin tx message and the peer it came from together +// so the block handler has access to that information. +type txMsg struct { + tx *core.Transaction + peer *peer.Peer + reply chan struct{} +} + +// getSyncPeerMsg is a message type to be sent across the message channel for +// retrieving the current sync peer. +type getSyncPeerMsg struct { + reply chan uint64 +} + +// isCurrentMsg is a message type to be sent across the message channel for +// requesting whether or not the sync manager believes it is synced with the +// currently connected peers. +type isCurrentMsg struct { + reply chan bool +} + +// pauseMsg is a message type to be sent across the message channel for +// pausing the sync manager. This effectively provides the caller with +// exclusive access over the manager until a receive is performed on the +// unpause channel. +type pauseMsg struct { + unpause <-chan struct{} +} + +// peerSyncState stores additional information that the SyncManager tracks +// about a peer. +type peerSyncState struct { + syncCandidate bool + requestQueue []*msg.InvVect + requestedTxns map[common.Uint256]struct{} + requestedBlocks map[common.Uint256]struct{} + receivedBlocks uint32 + badBlocks uint32 + receivedTxs uint32 + falsePositives uint32 +} + +func (s *peerSyncState) badBlockRate() float64 { + return float64(s.badBlocks) / float64(s.receivedBlocks) +} + +func (s *peerSyncState) falsePosRate() float64 { + return float64(s.falsePositives) / float64(s.receivedTxs) +} + +// SyncManager is used to communicate block related messages with peers. The +// SyncManager is started as by executing Start() in a goroutine. Once started, +// it selects peers to sync from and starts the initial block download. Once the +// chain is in sync, the SyncManager handles incoming block and header +// notifications and relays announcements of new blocks to peers. +type SyncManager struct { + started int32 + shutdown int32 + cfg Config + msgChan chan interface{} + wg sync.WaitGroup + quit chan struct{} + + // These fields should only be accessed from the blockHandler thread + requestedTxns map[common.Uint256]struct{} + requestedBlocks map[common.Uint256]struct{} + txMemPool map[common.Uint256]struct{} + syncPeer *peer.Peer + peerStates map[*peer.Peer]*peerSyncState +} + +// current returns true if we believe we are synced with our peers, false if we +// still have blocks to check +func (sm *SyncManager) current() bool { + // if blockChain thinks we are current and we have no syncPeer it + // is probably right. + if sm.syncPeer == nil { + return true + } + + // No matter what chain thinks, if we are below the block we are syncing + // to we are not current. + if sm.cfg.Chain.BestHeight() < sm.syncPeer.Height() { + return false + } + return true +} + +// startSync will choose the best peer among the available candidate peers to +// download/sync the blockchain from. When syncing is already running, it +// simply returns. It also examines the candidates for any which are no longer +// candidates and removes them as needed. +func (sm *SyncManager) startSync() { + // Return if sync candidates less then MinPeersForSync. + if len(sm.getSyncCandidates()) < sm.cfg.MinPeersForSync { + return + } + + // Return now if we're already syncing. + if sm.syncPeer != nil { + return + } + + bestHeight := sm.cfg.Chain.BestHeight() + var bestPeer *peer.Peer + for peer, state := range sm.peerStates { + if !state.syncCandidate { + continue + } + + // Remove sync candidate peers that are no longer candidates due + // to passing their latest known block. NOTE: The < is + // intentional as opposed to <=. While technically the peer + // doesn't have a later block when it's equal, it will likely + // have one soon so it is a reasonable choice. It also allows + // the case where both are at 0 such as during regression test. + if peer.Height() < bestHeight { + state.syncCandidate = false + continue + } + + // Pick the first available candidate. + if bestPeer == nil { + bestPeer = peer + continue + } + + // Pick the highest available candidate. + if peer.Height() > bestPeer.Height() { + bestPeer = peer + } + } + + // Start syncing from the best peer if one was selected. + if bestPeer != nil { + sm.syncWith(bestPeer) + } else { + log.Warnf("No sync peer candidates available") + } +} + +func (sm *SyncManager) syncWith(p *peer.Peer) { + // Clear the requestedBlocks if the sync peer changes, otherwise we + // may ignore blocks we need that the last sync peer failed to send. + sm.requestedBlocks = make(map[common.Uint256]struct{}) + + log.Infof("Syncing to block height %d from peer %v", p.Height(), p.Addr()) + + locator := sm.cfg.Chain.LatestBlockLocator() + p.PushGetBlocksMsg(locator, &zeroHash) + sm.syncPeer = p +} + +// isSyncCandidate returns whether or not the peer is a candidate to consider +// syncing from. +func (sm *SyncManager) isSyncCandidate(peer *peer.Peer) bool { + services := peer.Services() + // Candidate if all checks passed. + return services&p2p.SFNodeNetwork == p2p.SFNodeNetwork && + services&p2p.SFNodeBloom == p2p.SFNodeBloom +} + +// getSyncCandidates returns the peers that are sync candidate. +func (sm *SyncManager) getSyncCandidates() []*peer.Peer { + candidates := make([]*peer.Peer, 0, len(sm.peerStates)) + for peer, state := range sm.peerStates { + candidate := *peer + if state.syncCandidate { + candidates = append(candidates, &candidate) + } + } + return candidates +} + +// updateBloomFilter update the bloom filter and send it to the given peer. +func (sm *SyncManager) updateBloomFilter(p *peer.Peer) { + msg := sm.cfg.UpdateFilter().GetFilterLoadMsg() + doneChan := make(chan struct{}) + p.QueueMessage(msg, doneChan) + + go func(p *peer.Peer) { + select { + case <-doneChan: + // Reset false positive state. + state, ok := sm.peerStates[p] + if ok { + state.receivedTxs = 0 + state.falsePositives = 0 + } + + case <-p.Quit(): + return + } + }(p) +} + +// handleNewPeerMsg deals with new peers that have signalled they may +// be considered as a sync peer (they have already successfully negotiated). It +// also starts syncing if needed. It is invoked from the syncHandler goroutine. +func (sm *SyncManager) handleNewPeerMsg(peer *peer.Peer) { + // Ignore if in the process of shutting down. + if atomic.LoadInt32(&sm.shutdown) != 0 { + return + } + + log.Infof("New valid peer %s", peer) + + // Initialize the peer state + isSyncCandidate := sm.isSyncCandidate(peer) + sm.peerStates[peer] = &peerSyncState{ + syncCandidate: isSyncCandidate, + requestedTxns: make(map[common.Uint256]struct{}), + requestedBlocks: make(map[common.Uint256]struct{}), + } + + // Start syncing by choosing the best candidate if needed. + if isSyncCandidate && sm.syncPeer == nil { + sm.startSync() + } +} + +// handleDonePeerMsg deals with peers that have signalled they are done. It +// removes the peer as a candidate for syncing and in the case where it was +// the current sync peer, attempts to select a new best peer to sync from. It +// is invoked from the syncHandler goroutine. +func (sm *SyncManager) handleDonePeerMsg(peer *peer.Peer) { + state, exists := sm.peerStates[peer] + if !exists { + log.Warnf("Received done peer message for unknown peer %s", peer) + return + } + + // Remove the peer from the list of candidate peers. + delete(sm.peerStates, peer) + + log.Infof("Lost peer %s", peer) + + // Remove requested transactions from the global map so that they will + // be fetched from elsewhere next time we get an inv. + for txHash := range state.requestedTxns { + delete(sm.requestedTxns, txHash) + } + + // Remove requested blocks from the global map so that they will be + // fetched from elsewhere next time we get an inv. + for blockHash := range state.requestedBlocks { + delete(sm.requestedBlocks, blockHash) + } + + // Attempt to find a new peer to sync from if the quitting peer is the + // sync peer. + if sm.syncPeer == peer { + sm.syncPeer = nil + sm.startSync() + } +} + +// handleTxMsg handles transaction messages from all peers. +func (sm *SyncManager) handleTxMsg(tmsg *txMsg) { + peer := tmsg.peer + state, exists := sm.peerStates[peer] + if !exists { + log.Warnf("Received tx message from unknown peer %s", peer) + return + } + + txHash := tmsg.tx.Hash() + + _, ok := state.requestedTxns[txHash] + if !ok { + log.Warnf("Peer %s is sending us transactions we didn't request", peer) + peer.Disconnect() + return + } + sm.txMemPool[txHash] = struct{}{} + + // Remove transaction from request maps. Either the mempool/chain + // already knows about it and as such we shouldn't have any more + // instances of trying to fetch it, or we failed to insert and thus + // we'll retry next time we get an inv. + delete(state.requestedTxns, txHash) + delete(sm.requestedTxns, txHash) + + fp, err := sm.cfg.Chain.CommitTx(tmsg.tx) + if err != nil { + log.Errorf("commit transaction error %v", err) + } + + if fp { + log.Debugf("Tx %s from Peer%d is a false positive.", txHash.String(), peer.ID()) + state.falsePositives++ + if state.falsePosRate() > maxFalsePositiveRate { + sm.updateBloomFilter(peer) + } + } + +} + +// handleBlockMsg handles block messages from all peers. Blocks are requested +// in response to inv packets both during initial sync and after. +func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { + peer := bmsg.peer + + // We don't need to process blocks when we're syncing. They wont connect anyway + if peer != sm.syncPeer && !sm.current() { + log.Warnf("Received block from %s when we aren't current", peer) + return + } + state, exists := sm.peerStates[peer] + if !exists { + log.Warnf("Received block message from unknown peer %s", peer) + peer.Disconnect() + return + } + + // If we didn't ask for this block then the peer is misbehaving. + block := bmsg.block + header := block.Header + blockHash := header.Hash() + if _, exists = state.requestedBlocks[blockHash]; !exists { + peer.Disconnect() + return + } + + // Remove block from request maps. Either chain will know about it and + // so we shouldn't have any more instances of trying to fetch it, or we + // will fail the insert and thus we'll retry next time we get an inv. + state.receivedBlocks++ + delete(state.requestedBlocks, blockHash) + delete(sm.requestedBlocks, blockHash) + + newBlock, reorg, newHeight, fps, err := sm.cfg.Chain.CommitBlock(block) + // If this is an orphan block which doesn't connect to the chain, it's possible + // that we might be synced on the longest chain, but not the most-work chain like + // we should be. To make sure this isn't the case, let's sync from the peer who + // sent us this orphan block. + if err == blockchain.OrphanBlockError && sm.current() { + log.Debug("Received orphan header, checking peer for more blocks") + state.requestQueue = []*msg.InvVect{} + state.requestedBlocks = make(map[common.Uint256]struct{}) + sm.requestedBlocks = make(map[common.Uint256]struct{}) + sm.syncWith(peer) + return + } + + // The sync peer sent us an orphan header in the middle of a sync. This could + // just be the last block in the batch which represents the tip of the chain. + // In either case let's adjust the score for this peer downwards. If it goes + // negative it means he's slamming us with blocks that don't fit in our chain + // so disconnect. + if err == blockchain.OrphanBlockError && !sm.current() { + state.badBlocks++ + if state.badBlockRate() > maxBadBlockRate { + log.Warnf("Disconnecting from peer %s because he sent us too many bad blocks", peer) + peer.Disconnect() + return + } + log.Warnf("Received unrequested block from peer %s", peer) + return + } + + // Log other error message and return. + if err != nil { + log.Error(err) + return + } + + // Check false positive rate. + state.falsePositives += fps + if state.falsePosRate() > maxFalsePositiveRate { + + } + + // We can exit here if the block is already known + if !newBlock { + log.Debugf("Received duplicate block %s", blockHash.String()) + return + } + + log.Infof("Received block %s at height %d", blockHash.String(), newHeight) + + // Check reorg + if reorg && sm.current() { + // Clear request state for new sync + state.requestQueue = []*msg.InvVect{} + state.requestedBlocks = make(map[common.Uint256]struct{}) + sm.requestedBlocks = make(map[common.Uint256]struct{}) + } + + // Clear mempool + sm.txMemPool = make(map[common.Uint256]struct{}) + + // If we're current now, nothing more to do. + if sm.current() { + peer.UpdateHeight(newHeight) + return + } + + // If we're not current and we've downloaded everything we've requested send another getblocks message. + // Otherwise we'll request the next block in the queue. + if len(state.requestQueue) == 0 { + locator := sm.cfg.Chain.LatestBlockLocator() + peer.PushGetBlocksMsg(locator, &zeroHash) + log.Debug("Request queue at zero. Pushing new locator.") + return + } + + // We have pending requests, so push a new getdata message. + sm.pushGetDataMsg(peer, state) +} + +// haveInventory returns whether or not the inventory represented by the passed +// inventory vector is known. This includes checking all of the various places +// inventory can be when it is in different states such as blocks that are part +// of the main chain, on a side chain, in the orphan pool, and transactions that +// are in the memory pool (either the main pool or orphan pool). +func (sm *SyncManager) haveInventory(invVect *msg.InvVect) bool { + switch invVect.Type { + case msg.InvTypeBlock: + fallthrough + case msg.InvTypeFilteredBlock: + // Ask chain if the block is known to it in any form (main + // chain, side chain, or orphan). + return sm.cfg.Chain.HaveBlock(&invVect.Hash) + + case msg.InvTypeTx: + // Is transaction already in mempool + _, ok := sm.txMemPool[invVect.Hash] + return ok + } + + // The requested inventory is is an unsupported type, so just claim + // it is known to avoid requesting it. + return true +} + +// handleInvMsg handles inv messages from all peers. +// We examine the inventory advertised by the remote peer and act accordingly. +func (sm *SyncManager) handleInvMsg(imsg *invMsg) { + peer := imsg.peer + state, exists := sm.peerStates[peer] + if !exists { + log.Warnf("Received inv message from unknown peer %s", peer) + return + } + + invVects := imsg.inv.InvList + + // Ignore invs from peers that aren't the sync if we are not current. + // Helps prevent fetching a mass of orphans. + if peer != sm.syncPeer && !sm.current() { + return + } + + // Request the advertised inventory if we don't already have it. + for _, iv := range invVects { + // Ignore unsupported inventory types. + switch iv.Type { + case msg.InvTypeBlock: + iv.Type = msg.InvTypeFilteredBlock + case msg.InvTypeTx: + default: + continue + } + + // Request the inventory if we don't already have it. + if !sm.haveInventory(iv) { + // Add it to the request queue. + state.requestQueue = append(state.requestQueue, iv) + continue + } + } + + sm.pushGetDataMsg(peer, state) +} + +func (sm *SyncManager) pushGetDataMsg(peer *peer.Peer, state *peerSyncState) { + // Request as much as possible at once. Anything that won't fit into + // the request will be requested on the next inv message. + numRequested := 0 + gdmsg := msg.NewGetData() + requestQueue := state.requestQueue + for len(requestQueue) != 0 { + iv := requestQueue[0] + requestQueue[0] = nil + requestQueue = requestQueue[1:] + + switch iv.Type { + case msg.InvTypeFilteredBlock: + // Request the block if there is not already a pending + // request. + if _, exists := sm.requestedBlocks[iv.Hash]; !exists { + sm.requestedBlocks[iv.Hash] = struct{}{} + sm.limitMap(sm.requestedBlocks, maxRequestedBlocks) + state.requestedBlocks[iv.Hash] = struct{}{} + + gdmsg.AddInvVect(iv) + numRequested++ + } + + case msg.InvTypeTx: + // Request the transaction if there is not already a + // pending request. + if _, exists := sm.requestedTxns[iv.Hash]; !exists { + sm.requestedTxns[iv.Hash] = struct{}{} + sm.limitMap(sm.requestedTxns, maxRequestedTxns) + state.requestedTxns[iv.Hash] = struct{}{} + + gdmsg.AddInvVect(iv) + numRequested++ + } + } + + if numRequested >= msg.MaxInvPerMsg { + break + } + } + state.requestQueue = requestQueue + if len(gdmsg.InvList) > 0 { + peer.QueueMessage(gdmsg, nil) + } +} + +// limitMap is a helper function for maps that require a maximum limit by +// evicting a random transaction if adding a new value would cause it to +// overflow the maximum allowed. +func (sm *SyncManager) limitMap(m map[common.Uint256]struct{}, limit int) { + if len(m)+1 > limit { + // Remove a random entry from the map. For most compilers, Go's + // range statement iterates starting at a random item although + // that is not 100% guaranteed by the spec. The iteration order + // is not important here because an adversary would have to be + // able to pull off preimage attacks on the hashing function in + // order to target eviction of specific entries anyways. + for txHash := range m { + delete(m, txHash) + return + } + } +} + +// blockHandler is the main handler for the sync manager. It must be run as a +// goroutine. It processes block and inv messages in a separate goroutine +// from the peer handlers so the block (MsgBlock) messages are handled by a +// single thread without needing to lock memory data structures. This is +// important because the sync manager controls which blocks are needed and how +// the fetching should proceed. +func (sm *SyncManager) blockHandler() { +out: + for { + select { + case m := <-sm.msgChan: + switch msg := m.(type) { + case *newPeerMsg: + sm.handleNewPeerMsg(msg.peer) + + case *txMsg: + sm.handleTxMsg(msg) + msg.reply <- struct{}{} + + case *blockMsg: + sm.handleBlockMsg(msg) + msg.reply <- struct{}{} + + case *invMsg: + sm.handleInvMsg(msg) + + case *donePeerMsg: + sm.handleDonePeerMsg(msg.peer) + + case getSyncPeerMsg: + var peerID uint64 + if sm.syncPeer != nil { + peerID = sm.syncPeer.ID() + } + msg.reply <- peerID + + case isCurrentMsg: + msg.reply <- sm.current() + + case pauseMsg: + // Wait until the sender unpauses the manager. + <-msg.unpause + + default: + log.Warnf("Invalid message type in block "+ + "handler: %T", msg) + } + + case <-sm.quit: + break out + } + } + + sm.wg.Done() + log.Trace("Block handler done") +} + +// NewPeer informs the sync manager of a newly active peer. +func (sm *SyncManager) NewPeer(peer *peer.Peer) { + // Ignore if we are shutting down. + if atomic.LoadInt32(&sm.shutdown) != 0 { + return + } + sm.msgChan <- &newPeerMsg{peer: peer} +} + +// QueueTx adds the passed transaction message and peer to the block handling +// queue. Responds to the done channel argument after the tx message is +// processed. +func (sm *SyncManager) QueueTx(tx *core.Transaction, peer *peer.Peer, done chan struct{}) { + // Don't accept more transactions if we're shutting down. + if atomic.LoadInt32(&sm.shutdown) != 0 { + done <- struct{}{} + return + } + + sm.msgChan <- &txMsg{tx: tx, peer: peer, reply: done} +} + +// QueueBlock adds the passed block message and peer to the block handling +// queue. Responds to the done channel argument after the block message is +// processed. +func (sm *SyncManager) QueueBlock(block *util.Block, peer *peer.Peer, done chan struct{}) { + // Don't accept more blocks if we're shutting down. + if atomic.LoadInt32(&sm.shutdown) != 0 { + done <- struct{}{} + return + } + + sm.msgChan <- &blockMsg{block: block, peer: peer, reply: done} +} + +// QueueInv adds the passed inv message and peer to the block handling queue. +func (sm *SyncManager) QueueInv(inv *msg.Inv, peer *peer.Peer) { + // No channel handling here because peers do not need to block on inv + // messages. + if atomic.LoadInt32(&sm.shutdown) != 0 { + return + } + + sm.msgChan <- &invMsg{inv: inv, peer: peer} +} + +// DonePeer informs the blockmanager that a peer has disconnected. +func (sm *SyncManager) DonePeer(peer *peer.Peer) { + // Ignore if we are shutting down. + if atomic.LoadInt32(&sm.shutdown) != 0 { + return + } + + sm.msgChan <- &donePeerMsg{peer: peer} +} + +// Start begins the core block handler which processes block and inv messages. +func (sm *SyncManager) Start() { + // Already started? + if atomic.AddInt32(&sm.started, 1) != 1 { + return + } + + log.Trace("Starting sync manager") + sm.wg.Add(1) + go sm.blockHandler() +} + +// Stop gracefully shuts down the sync manager by stopping all asynchronous +// handlers and waiting for them to finish. +func (sm *SyncManager) Stop() error { + if atomic.AddInt32(&sm.shutdown, 1) != 1 { + log.Warnf("Sync manager is already in the process of " + + "shutting down") + return nil + } + + log.Infof("Sync manager shutting down") + close(sm.quit) + sm.wg.Wait() + return nil +} + +// SyncPeerID returns the ID of the current sync peer, or 0 if there is none. +func (sm *SyncManager) SyncPeerID() uint64 { + reply := make(chan uint64) + sm.msgChan <- getSyncPeerMsg{reply: reply} + return <-reply +} + +// IsCurrent returns whether or not the sync manager believes it is synced with +// the connected peers. +func (sm *SyncManager) IsCurrent() bool { + reply := make(chan bool) + sm.msgChan <- isCurrentMsg{reply: reply} + return <-reply +} + +// Pause pauses the sync manager until the returned channel is closed. +// +// Note that while paused, all peer and block processing is halted. The +// message sender should avoid pausing the sync manager for long durations. +func (sm *SyncManager) Pause() chan<- struct{} { + c := make(chan struct{}) + sm.msgChan <- pauseMsg{c} + return c +} + +// New constructs a new SyncManager. Use Start to begin processing asynchronous +// block, tx, and inv updates. +func New(cfg *Config) (*SyncManager, error) { + sm := SyncManager{ + cfg: *cfg, + txMemPool: make(map[common.Uint256]struct{}), + requestedTxns: make(map[common.Uint256]struct{}), + requestedBlocks: make(map[common.Uint256]struct{}), + peerStates: make(map[*peer.Peer]*peerSyncState), + msgChan: make(chan interface{}, cfg.MaxPeers*3), + quit: make(chan struct{}), + } + + return &sm, nil +} From fddabefbf8e5c9fac46710eaed6d84d077b14182 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 8 Sep 2018 13:34:55 +0800 Subject: [PATCH 08/73] initial util package implement --- util/block.go | 11 +++++++++++ util/header.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- util/tx.go | 13 +++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 util/block.go create mode 100644 util/tx.go diff --git a/util/block.go b/util/block.go new file mode 100644 index 0000000..704c2b0 --- /dev/null +++ b/util/block.go @@ -0,0 +1,11 @@ +package util + +// Block represent a block that stored into +// blockchain database. +type Block struct { + // The store header of this block. + *Header + + // Transactions of this block. + Transactions []*Tx +} diff --git a/util/header.go b/util/header.go index ff8d7d5..bb0dca9 100644 --- a/util/header.go +++ b/util/header.go @@ -1,7 +1,8 @@ -package database +package util import ( "bytes" + "github.com/elastos/Elastos.ELA.Utility/common" "math/big" "github.com/elastos/Elastos.ELA/core" @@ -12,6 +13,11 @@ type Header struct { // The origin header of the block *core.Header + // MerkleProof for transactions packed in this block + NumTxs uint32 + Hashes []*common.Uint256 + Flags []byte + // The total work from the genesis block to this // current block TotalWork *big.Int @@ -24,6 +30,26 @@ func (sh *Header) Serialize() ([]byte, error) { return nil, err } + err = common.WriteUint32(buf, sh.NumTxs) + if err != nil { + return nil, err + } + + err = common.WriteVarUint(buf, uint64(len(sh.Hashes))) + if err != nil { + return nil, err + } + + err = common.WriteElement(buf, sh.Hashes) + if err != nil { + return nil, err + } + + err = common.WriteVarBytes(buf, sh.Flags) + if err != nil { + return nil, err + } + biBytes := sh.TotalWork.Bytes() pad := make([]byte, 32-len(biBytes)) serializedBI := append(pad, biBytes...) @@ -38,6 +64,27 @@ func (sh *Header) Deserialize(b []byte) error { return err } + sh.NumTxs, err = common.ReadUint32(r) + if err != nil { + return err + } + + count, err := common.ReadVarUint(r, 0) + if err != nil { + return err + } + + sh.Hashes = make([]*common.Uint256, count) + err = common.ReadElement(r, &sh.Hashes) + if err != nil { + return err + } + + sh.Flags, err = common.ReadVarBytes(r) + if err != nil { + return err + } + biBytes := make([]byte, 32) _, err = r.Read(biBytes) if err != nil { diff --git a/util/tx.go b/util/tx.go new file mode 100644 index 0000000..765180f --- /dev/null +++ b/util/tx.go @@ -0,0 +1,13 @@ +package util + +import "github.com/elastos/Elastos.ELA/core" + +// Tx is a data structure used in database. +type Tx struct { + // The origin transaction data. + *core.Transaction + + // The block height that this transaction + // belongs to. + Height uint32 +} \ No newline at end of file From b006d7adc9031fef62d2a7d1ba8fbdda02f8bd70 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 8 Sep 2018 13:36:03 +0800 Subject: [PATCH 09/73] adjust sdk inteface and spvservice implement --- sdk/interface.go | 96 +++++--- sdk/spvservice.go | 617 ++++++++++++++++++++++++++++------------------ 2 files changed, 438 insertions(+), 275 deletions(-) diff --git a/sdk/interface.go b/sdk/interface.go index 3ae667e..db1eef1 100644 --- a/sdk/interface.go +++ b/sdk/interface.go @@ -1,44 +1,75 @@ package sdk import ( - "github.com/elastos/Elastos.ELA.SPV/store" - "github.com/elastos/Elastos.ELA.Utility/p2p/server" - - "github.com/elastos/Elastos.ELA.SPV/net" + "github.com/elastos/Elastos.ELA.SPV/database" + "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - ela "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/core" ) /* -SPV service is a high level implementation with all SPV logic implemented. -SPV service is extend from SPV client and implement BlockChain and block synchronize on it. -With SPV service, you just need to implement your own HeaderStore and SPVServiceConfig, and let other stuff go. +IService is an implementation for SPV features. */ -type SPVService interface { +type IService interface { // Start SPV service Start() // Stop SPV service Stop() - // IsSyncing returns the current state of the service, to indicate that the service - // is in syncing mode or waiting mode. - IsSyncing() bool + // IsCurrent returns whether or not the SPV service believes it is synced with + // the connected peers. + IsCurrent() bool + + // UpdateFilter is a trigger to make SPV service refresh the current + // transaction filer(in our implementation the bloom filter) and broadcast the + // new filter to connected peers. This will invoke the GetFilterData() method + // in Config. + UpdateFilter() // SendTransaction broadcast a transaction message to the peer to peer network. - SendTransaction(ela.Transaction) (*common.Uint256, error) + SendTransaction(core.Transaction) error } -type SPVServiceConfig struct { - // The server access into blockchain peer to peer network - Server server.IServer +// StateNotifier exposes methods to notify status changes of transactions and blocks. +type StateNotifier interface { + // TransactionAccepted will be invoked after a transaction sent by + // SendTransaction() method has been accepted. Notice: this method needs at + // lest two connected peers to work. + TransactionAccepted(tx *util.Tx) + + // TransactionRejected will be invoked if a transaction sent by SendTransaction() + // method has been rejected. + TransactionRejected(tx *util.Tx) + + // TransactionConfirmed will be invoked after a transaction sent by + // SendTransaction() method has been packed into a block. + TransactionConfirmed(tx *util.Tx) + + // BlockCommitted will be invoked when a block and transactions within it are + // successfully committed into database. + BlockCommitted(block *util.Block) +} + +// Config is the configuration settings to the SPV service. +type Config struct { + // The magic number to indicate which network to access. + Magic uint32 + + // The seed peers addresses in [host:port] or [ip:port] format. + SeedList []string + + // The max peer connections. + MaxPeers int + + // The min candidate peers count to start syncing progress. + MinPeersForSync int // Foundation address of the current access blockhain network Foundation string // The database to store all block headers - HeaderStore store.HeaderStore + ChainStore database.ChainStore // GetFilterData() returns two arguments. // First arguments are all addresses stored in your data store. @@ -48,34 +79,19 @@ type SPVServiceConfig struct { // reference of an transaction output. If an address ever received an transaction output, // there will be the outpoint reference to it. Any time you want to spend the balance of an // address, you must provide the reference of the balance which is an outpoint in the transaction input. - GetFilterData func() ([]*common.Uint168, []*ela.OutPoint) - - // When interested transactions received, this method will call back them. - // The height is the block height where this transaction has been packed. - // Returns if the transaction is a match, for there will be transactions that - // are not interested go through this method. If a transaction is not a match - // return false as a false positive mark. If anything goes wrong, return error. - // Notice: this method will be callback when commit block - CommitTx func(tx *ela.Transaction, height uint32) (bool, error) + GetFilterData func() ([]*common.Uint168, []*core.OutPoint) - // This method will be callback after a block and transactions with it are - // successfully committed into database. - OnBlockCommitted func(*msg.MerkleBlock, []*ela.Transaction) - - // When the blockchain meet a reorganization, data should be rollback to the fork point. - // The Rollback method will callback the current rollback height, for example OnChainRollback(100) - // means data on height 100 has been deleted, current chain height will be 99. You should rollback - // stored data including UTXOs STXOs Txs etc. according to the given height. - // If anything goes wrong, return an error. - OnRollback func(height uint32) error + // StateNotifier is an optional config, if you don't want to receive state changes of transactions + // or blocks, just keep it blank. + StateNotifier StateNotifier } /* -Get a SPV service instance. +NewService returns a new SPV service instance. there are two implementations you need to do, DataStore and GetBloomFilter() method. DataStore is an interface including all methods you need to implement placed in db/datastore.go. Also an sample APP spvwallet is contain in this project placed in spvwallet folder. */ -func GetSPVService(config SPVServiceConfig) (SPVService, error) { - return NewSPVServiceImpl(config) +func NewService(config *Config) (IService, error) { + return NewSPVService(config) } diff --git a/sdk/spvservice.go b/sdk/spvservice.go index 96c0dee..e9b4cb7 100644 --- a/sdk/spvservice.go +++ b/sdk/spvservice.go @@ -1,335 +1,482 @@ package sdk import ( - "errors" "fmt" "time" + "github.com/elastos/Elastos.ELA.SPV/blockchain" "github.com/elastos/Elastos.ELA.SPV/log" - "github.com/elastos/Elastos.ELA.SPV/net" + spvpeer "github.com/elastos/Elastos.ELA.SPV/peer" + "github.com/elastos/Elastos.ELA.SPV/sync" + "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA.Utility/p2p/peer" + "github.com/elastos/Elastos.ELA.Utility/p2p/server" "github.com/elastos/Elastos.ELA/bloom" "github.com/elastos/Elastos.ELA/core" ) const ( - SendTxTimeout = time.Second * 10 + TxExpireTime = time.Hour * 24 + TxRebroadcastDuration = time.Minute * 15 ) +type sendTxMsg struct { + tx *core.Transaction + expire time.Time +} + +type txInvMsg struct { + iv *msg.InvVect +} + +type txRejectMsg struct { + iv *msg.InvVect +} + +type blockMsg struct { + block *util.Block +} + // The SPV service implementation -type SPVServiceImpl struct { - *net.ServerPeer - syncManager *SyncManager - chain *BlockChain - pendingTx common.Uint256 - txAccept chan *common.Uint256 - txReject chan *msg.Reject - config SPVServiceConfig +type service struct { + server.IServer + cfg Config + syncManager *sync.SyncManager + + newPeerQueue chan *peer.Peer + donePeerQueue chan *peer.Peer + txQueue chan interface{} + quit chan struct{} } // Create a instance of SPV service implementation. -func NewSPVServiceImpl(config SPVServiceConfig) (*SPVServiceImpl, error) { +func NewSPVService(cfg *Config) (*service, error) { // Initialize blockchain - chain, err := NewBlockchain(config.Foundation, config.HeaderStore) + chain, err := blockchain.New(cfg.Foundation, cfg.ChainStore) if err != nil { return nil, err } // Create SPV service instance - service := &SPVServiceImpl{ - ServerPeer: config.Server, - chain: chain, - config: config, + service := &service{ + cfg: *cfg, + newPeerQueue: make(chan *peer.Peer), + donePeerQueue: make(chan *peer.Peer), + txQueue: make(chan interface{}, 3), + quit: make(chan struct{}), } - // Create sync manager config - syncConfig := SyncManageConfig{ - LocalHeight: chain.BestHeight, - GetBlocks: service.GetBlocks, + var maxPeers int + var minPeersForSync int + if cfg.MaxPeers > 0 { + maxPeers = cfg.MaxPeers + } + if cfg.MinPeersForSync > 0 { + minPeersForSync = cfg.MinPeersForSync + } + if cfg.MinPeersForSync > cfg.MaxPeers { + minPeersForSync = cfg.MaxPeers } - service.syncManager = NewSyncManager(syncConfig) - - // Set manage config - service.SetConfig(net.PeerManageConfig{ - OnHandshake: service.OnHandshake, - OnPeerEstablish: service.OnPeerEstablish, - }) + // Create sync manager instance. + syncCfg := sync.NewDefaultConfig(chain, service.updateFilter) + syncCfg.MaxPeers = maxPeers + syncCfg.MinPeersForSync = minPeersForSync + syncManager, err := sync.New(syncCfg) + if err != nil { + return nil, err + } + service.syncManager = syncManager + + // Initiate P2P server configuration + serverCfg := server.NewDefaultConfig( + cfg.Magic, + cfg.SeedList, + nil, + service.newPeer, + service.donePeer, + service.makeEmptyMessage, + func() uint64 { return uint64(chain.BestHeight()) }, + ) + serverCfg.MaxPeers = maxPeers + + // Create P2P server. + server, err := server.NewServer(serverCfg) + if err != nil { + return nil, err + } + service.IServer = server return service, nil } -func (s *SPVServiceImpl) OnHandshake(v *msg.Version) error { - if v.Services/OpenService&1 == 0 { - return errors.New("SPV service not enabled on connected peer") - } +func (s *service) start() { + go s.peersHandler() + go s.txHandler() +} - return nil +func (s *service) updateFilter() *bloom.Filter { + return BuildBloomFilter(s.cfg.GetFilterData()) } -func (s *SPVServiceImpl) OnPeerEstablish(peer *net.Peer) { - // Create spv peer config - config := SPVPeerConfig{ - LocalHeight: s.LocalHeight, - OnInventory: s.OnInventory, - OnMerkleBlock: s.OnMerkleBlock, - OnTx: s.OnTx, - OnNotFound: s.OnNotFound, - OnReject: s.OnReject, - } +func (s *service) makeEmptyMessage(cmd string) (p2p.Message, error) { + var message p2p.Message + switch cmd { + case p2p.CmdVersion: + message = new(msg.Version) - s.syncManager.AddNeighborPeer(NewSPVPeer(peer, config)) + case p2p.CmdVerAck: + message = new(msg.VerAck) - // Load bloom filter - doneChan := make(chan struct{}) - peer.QueueMessage(s.BloomFilter(), doneChan) - <-doneChan -} + case p2p.CmdGetAddr: + message = new(msg.GetAddr) -func (s *SPVServiceImpl) LocalHeight() uint32 { - return uint32(s.ServerPeer.Height()) -} + case p2p.CmdAddr: + message = new(msg.Addr) -func (s *SPVServiceImpl) Start() { - s.ServerPeer.Start() - s.syncManager.start() - log.Info("SPV service started...") -} + case p2p.CmdInv: + message = new(msg.Inv) -func (s *SPVServiceImpl) Stop() { - s.chain.Close() - log.Info("SPV service stopped...") -} + case p2p.CmdGetData: + message = new(msg.GetData) -func (s *SPVServiceImpl) ChainState() ChainState { - return s.chain.state -} + case p2p.CmdNotFound: + message = new(msg.NotFound) -func (s *SPVServiceImpl) ReloadFilter() { - log.Debug() - s.Broadcast(BuildBloomFilter(s.config.GetFilterData()).GetFilterLoadMsg()) -} + case p2p.CmdTx: + message = msg.NewTx(new(core.Transaction)) -func (s *SPVServiceImpl) SendTransaction(tx core.Transaction) (*common.Uint256, error) { - log.Debug() + case p2p.CmdPing: + message = new(msg.Ping) - if s.GetNeighborCount() == 0 { - return nil, fmt.Errorf("method not available, no peers connected") - } + case p2p.CmdPong: + message = new(msg.Pong) - s.txAccept = make(chan *common.Uint256, 1) - s.txReject = make(chan *msg.Reject, 1) + case p2p.CmdMerkleBlock: + message = msg.NewMerkleBlock(new(core.Header)) - finish := func() { - close(s.txAccept) - close(s.txReject) - s.txAccept = nil - s.txReject = nil - } - // Set transaction in pending - s.pendingTx = tx.Hash() - // Broadcast transaction to neighbor peers - s.Broadcast(msg.NewTx(&tx)) - // Query neighbors mempool see if transaction was successfully added to mempool - s.Broadcast(new(msg.MemPool)) - - // Wait for result - timer := time.NewTimer(SendTxTimeout) - select { - case <-timer.C: - finish() - return nil, fmt.Errorf("Send transaction timeout") - case <-s.txAccept: - timer.Stop() - finish() - // commit unconfirmed transaction to db - _, err := s.config.CommitTx(&tx, 0) - return &s.pendingTx, err - case msg := <-s.txReject: - timer.Stop() - finish() - return nil, fmt.Errorf("Transaction rejected Code: %s, Reason: %s", msg.Code.String(), msg.Reason) + case p2p.CmdReject: + message = new(msg.Reject) + + default: + return nil, fmt.Errorf("unhandled command [%s]", cmd) } + return message, nil } -func (s *SPVServiceImpl) GetBlocks() *msg.GetBlocks { - // Get blocks returns a inventory message which contains block hashes - locator := s.chain.GetBlockLocatorHashes() - return msg.NewGetBlocks(locator, common.EmptyHash) +func (s *service) newPeer(peer *peer.Peer) { + s.newPeerQueue <- peer } -func (s *SPVServiceImpl) BloomFilter() *msg.FilterLoad { - bloomFilter := BuildBloomFilter(s.config.GetFilterData()) - return bloomFilter.GetFilterLoadMsg() +func (s *service) donePeer(peer *peer.Peer) { + s.donePeerQueue <- peer } -func (s *SPVServiceImpl) OnInventory(peer *SPVPeer, m *msg.Inventory) error { - getData := msg.NewGetData() - - for _, inv := range m.InvList { - switch inv.Type { - case msg.InvTypeBlock: - // Filter duplicated block - if s.chain.IsKnownHeader(&inv.Hash) { +// peersHandler handles new peers and done peers from P2P server. +// When comes new peer, create a spv peer warpper for it +func (s *service) peersHandler() { + peers := make(map[*peer.Peer]*spvpeer.Peer) + +out: + for { + select { + case p := <-s.newPeerQueue: + // Create spv peer warpper for the new peer. + sp := spvpeer.NewPeer(p, + &spvpeer.Config{ + OnInv: s.onInv, + OnTx: s.onTx, + OnBlock: s.onBlock, + OnNotFound: s.onNotFound, + OnReject: s.onReject, + }) + + peers[p] = sp + s.syncManager.NewPeer(sp) + + case p := <-s.donePeerQueue: + sp, ok := peers[p] + if !ok { + log.Errorf("unknown done peer %v", p) continue } - // Kind of lame to send separate getData messages but this allows us - // to take advantage of the timeout on the upper layer. Otherwise we - // need separate timeout handling. - inv.Type = msg.InvTypeFilteredBlock - getData.AddInvVect(inv) - if s.syncManager.IsSyncPeer(peer) { - peer.blockQueue <- inv.Hash - } + s.syncManager.DonePeer(sp) - case msg.InvTypeTx: - if s.txAccept != nil && s.pendingTx.IsEqual(inv.Hash) { - s.txAccept <- nil - continue - } - getData.AddInvVect(inv) - peer.EnqueueTx(inv.Hash) - - default: - continue + case <-s.quit: + break out } } - if len(getData.InvList) > 0 { - peer.QueueMessage(getData) + // Drain any wait channels before we go away so we don't leave something + // waiting for us. +cleanup: + for { + select { + case <-s.newPeerQueue: + case <-s.donePeerQueue: + default: + break cleanup + } } - return nil + log.Trace("Service peers handler done") } -func (s *SPVServiceImpl) OnMerkleBlock(peer *SPVPeer, mBlock *msg.MerkleBlock) error { - blockHash := mBlock.Header.(*core.Header).Hash() +// txHandler handles transaction messages like send transaction, transaction inv +// transaction reject etc. +func (s *service) txHandler() { + var unconfirmed = make(map[common.Uint256]sendTxMsg) + var accepted = make(map[common.Uint256]struct{}) + var rejected = make(map[common.Uint256]struct{}) + + retryTicker := time.NewTicker(TxRebroadcastDuration) + defer retryTicker.Stop() + +out: + for { + select { + case tmsg := <-s.txQueue: + switch tmsg := tmsg.(type) { + case sendTxMsg: + txId := tmsg.tx.Hash() + tmsg.expire = time.Now().Add(TxExpireTime) + unconfirmed[txId] = tmsg + delete(accepted, txId) + delete(rejected, txId) + + // Broadcast unconfirmed transaction + s.IServer.BroadcastMessage(msg.NewTx(tmsg.tx)) + + case txInvMsg: + // When a transaction was accepted and add to the txMemPool, a + // txInv message will be received through message relay, but it + // only works when there are more than 2 peers connected. + txId := tmsg.iv.Hash + + // The transaction has been marked as accepted. + if _, ok := accepted[txId]; ok { + continue + } + + // The txInv is an unconfirmed transaction. + if txMsg, ok := unconfirmed[txId]; ok { + delete(unconfirmed, txId) + accepted[txId] = struct{}{} + + // Use a new goroutine do the invoke to prevent blocking. + go func(tx *core.Transaction) { + if s.cfg.StateNotifier != nil { + s.cfg.StateNotifier.TransactionAccepted( + &util.Tx{ + Transaction: tx, + Height: 0, + }) + } + }(txMsg.tx) + } + + case txRejectMsg: + // If some of the peers are bad actors, transaction can be both + // accepted and rejected. For we can not say who are bad actors + // and who are not, so just pick the first response and notify + // the transaction state change. + txId := tmsg.iv.Hash + + // The transaction has been marked as rejected. + if _, ok := rejected[txId]; ok { + continue + } + + // The txInv is an unconfirmed transaction. + if txMsg, ok := unconfirmed[txId]; ok { + rejected[txId] = struct{}{} + delete(unconfirmed, txId) + + // Use a new goroutine do the invoke to prevent blocking. + go func(tx *core.Transaction) { + if s.cfg.StateNotifier != nil { + s.cfg.StateNotifier.TransactionRejected( + &util.Tx{ + Transaction: tx, + Height: 0, + }) + } + }(txMsg.tx) + } + + case blockMsg: + // Loop through all packed transactions, see if match to any + // sent transactions. + confirmedTxs := make(map[util.Tx]struct{}) + for _, tx := range tmsg.block.Transactions { + txId := tx.Hash() + tx.Height = tmsg.block.Height + + if _, ok := unconfirmed[txId]; ok { + confirmedTxs[*tx] = struct{}{} + continue + } + + if _, ok := accepted[txId]; ok { + confirmedTxs[*tx] = struct{}{} + continue + } + + if _, ok := rejected[txId]; ok { + confirmedTxs[*tx] = struct{}{} + } + } + + for tx := range confirmedTxs { + txId := tx.Hash() + delete(unconfirmed, txId) + delete(accepted, txId) + delete(rejected, txId) + + // Use a new goroutine do the invoke to prevent blocking. + go func(tx *util.Tx) { + if s.cfg.StateNotifier != nil { + s.cfg.StateNotifier.TransactionConfirmed(tx) + } + }(&tx) + } + } + case <-retryTicker.C: + // Rebroadcast unconfirmed transactions. + now := time.Now() + for id, tx := range unconfirmed { + // Delete expired transaction. + if tx.expire.Before(now) { + delete(unconfirmed, id) + continue + } + + // Broadcast unconfirmed transaction + s.IServer.BroadcastMessage(msg.NewTx(tx.tx)) + } - // Merkleblock from sync peer - if s.syncManager.IsSyncPeer(peer) { - queueHash := <-peer.blockQueue - if !blockHash.IsEqual(queueHash) { - peer.Disconnect() - return fmt.Errorf("peer %d is sending us blocks out of order", peer.ID()) + case <-s.quit: + break out } } - txIds, err := bloom.CheckMerkleBlock(*mBlock) - if err != nil { - return fmt.Errorf("invalid merkleblock received %s", err.Error()) + // Drain any wait channels before we go away so we don't leave something + // waiting for us. +cleanup: + for { + select { + case <-s.txQueue: + default: + break cleanup + } } + log.Trace("Service transaction handler done") +} - dBlock := peer.EnqueueBlock(mBlock, txIds) - if dBlock != nil { - s.commitBlock(peer, dBlock) - - // Try continue sync progress - s.syncManager.ContinueSync() +func (s *service) SendTransaction(tx core.Transaction) error { + peersCount := s.IServer.ConnectedCount() + if peersCount < int32(s.cfg.MinPeersForSync) { + return fmt.Errorf("connected peers %d not enough for sending transactions", peersCount) + } + if !s.IsCurrent() { + return fmt.Errorf("spv service did not sync to current") } + s.txQueue <- sendTxMsg{tx: &tx} return nil } -func (s *SPVServiceImpl) OnTx(peer *SPVPeer, msg *msg.Tx) error { - tx := msg.Transaction.(*core.Transaction) - - obj, ok := peer.DequeueTx(tx) - if ok { - switch obj := obj.(type) { - case *block: - // commit block - s.commitBlock(peer, obj) - - // Try continue sync progress - s.syncManager.ContinueSync() - - case *core.Transaction: - // commit unconfirmed transaction - _, err := s.config.CommitTx(tx, 0) - if err == nil { - // Update bloom filter - peer.SendMessage(s.BloomFilter()) +func (s *service) onInv(sp *spvpeer.Peer, inv *msg.Inv) { + // If service already synced to current, it most likely to receive a relayed + // block or transaction inv, not a huge invList with block hashes. + if s.IsCurrent() { + for _, iv := range inv.InvList { + switch iv.Type { + case msg.InvTypeTx: + s.txQueue <- txInvMsg{iv: iv} } - - return err } } + s.syncManager.QueueInv(inv, sp) +} - return fmt.Errorf("Transaction not found in download queue %s", tx.Hash().String()) +func (s *service) onBlock(sp *spvpeer.Peer, block *util.Block) { + done := make(chan struct{}) + s.syncManager.QueueBlock(block, sp, done) + + // Use a new goroutine to prevent blocking. + go func() { + select { + case <-done: + s.txQueue <- blockMsg{block: block} + if s.cfg.StateNotifier != nil { + s.cfg.StateNotifier.BlockCommitted(block) + } + case <-sp.Quit(): + return + } + }() } -func (s *SPVServiceImpl) OnNotFound(peer *SPVPeer, notFound *msg.NotFound) error { - for _, iv := range notFound.InvList { - log.Warnf("Data not found type %s, hash %s", iv.Type.String(), iv.Hash.String()) - } - return nil +func (s *service) onTx(sp *spvpeer.Peer, tx *core.Transaction) { + done := make(chan struct{}) + s.syncManager.QueueTx(tx, sp, done) + <-done } -func (s *SPVServiceImpl) OnReject(peer *SPVPeer, msg *msg.Reject) error { - if s.pendingTx.IsEqual(msg.Hash); s.txReject != nil { - s.txReject <- msg - return nil - } - return fmt.Errorf("Received reject message from peer %d: Code: %s, Hash %s, Reason: %s", - peer.ID(), msg.Code.String(), msg.Hash.String(), msg.Reason) +func (s *service) onNotFound(sp *spvpeer.Peer, notFound *msg.NotFound) { + // Every thing we requested was came from this connected peer, so + // no reason it said I have some data you don't have and when you + // come to get it, it say oh I didn't have it. + log.Warnf("Peer %s is sending us notFound -- disconnecting", sp) + sp.Disconnect() } -func (s *SPVServiceImpl) commitBlock(peer *SPVPeer, block *block) { - header := block.Header.(*core.Header) - newTip, reorgFrom, err := s.chain.CommitHeader(*header) - if err != nil { - log.Errorf("Commit header failed %s", err.Error()) - return - } - if !newTip { - return +func (s *service) onReject(sp *spvpeer.Peer, reject *msg.Reject) { + if reject.Cmd == p2p.CmdTx { + s.txQueue <- txInvMsg{iv: &msg.InvVect{Type: msg.InvTypeTx, Hash: reject.Hash}} } + log.Warnf("reject message from peer %v: Code: %s, Hash %s, Reason: %s", + sp, reject.Code.String(), reject.Hash.String(), reject.Reason) +} - newHeight := s.chain.BestHeight() - if reorgFrom > 0 { - for i := reorgFrom; i > newHeight; i-- { - if err = s.config.OnRollback(i); err != nil { - log.Errorf("Rollback transaction at height %d failed %s", i, err.Error()) - return - } - } +func (s *service) IsCurrent() bool { + return s.syncManager.IsCurrent() +} - if !s.chain.IsSyncing() { - s.syncManager.StartSync() - return - } - } +func (s *service) UpdateFilter() { + // Update bloom filter + filter := s.updateFilter() - for _, tx := range block.txs { - // Increase received transaction count - peer.receivedTxs++ + // Broadcast filterload message to connected peers. + s.IServer.BroadcastMessage(filter.GetFilterLoadMsg()) +} - falsePositive, err := s.config.CommitTx(tx, header.Height) - if err != nil { - log.Errorf("Commit transaction %s failed %s", tx.Hash().String(), err.Error()) - return - } +func (s *service) Start() { + s.start() + s.syncManager.Start() + s.IServer.Start() + log.Info("SPV service started...") +} - // Increase false positive count - if falsePositive { - peer.fPositives++ - } +func (s *service) Stop() { + err := s.IServer.Stop() + if err != nil { + log.Error(err) } - // Refresh bloom filter if false positives meet target rate - if peer.GetFalsePositiveRate() > FalsePositiveRate { - // Reset false positives - peer.ResetFalsePositives() - - // Update bloom filter - peer.SendMessage(s.BloomFilter()) + err = s.syncManager.Stop() + if err != nil { + log.Error(err) } - s.ServerPeer.SetHeight(uint64(newHeight)) + s.cfg.ChainStore.Close() - // Notify block committed - go s.config.OnBlockCommitted(block.MerkleBlock, block.txs) + close(s.quit) + log.Info("SPV service stopped...") } From 087cef6bef21fae3b27a5a9dfa6c887f5a34572a Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 10 Sep 2018 17:28:45 +0800 Subject: [PATCH 10/73] adjust log levels, 0 info, 1 warn, 2 error, 3 fatal, 4 trace, 5 debug --- config.json.back | 11 +++ log/log.go | 176 +++++++++++++++-------------------------------- log/log_test.go | 2 +- 3 files changed, 68 insertions(+), 121 deletions(-) create mode 100644 config.json.back diff --git a/config.json.back b/config.json.back new file mode 100644 index 0000000..f366f89 --- /dev/null +++ b/config.json.back @@ -0,0 +1,11 @@ +{ + "Magic": 20180627, + "PrintLevel": 0, + "RPCPort": 20477, + "Foundation": "8ZNizBf4KhhPjeJRGpox6rPcHE5Np6tFx3", + "SeedList": [ + "node-regtest-002.elastos.org:22866", + "node-regtest-003.elastos.org:22866", + "node-regtest-004.elastos.org:22866" + ] +} \ No newline at end of file diff --git a/log/log.go b/log/log.go index acaea60..6b0871c 100644 --- a/log/log.go +++ b/log/log.go @@ -20,11 +20,12 @@ import ( ) const ( - Blue = "0;34" - Red = "0;31" - Green = "0;32" - Yellow = "0;33" - Cyan = "0;36" + White = "1;00" + Blue = "1;34" + Red = "1;31" + Green = "1;32" + Yellow = "1;33" + Cyan = "1;36" Pink = "1;35" ) @@ -33,29 +34,27 @@ func Color(code, msg string) string { } const ( - debugLog = iota - infoLog + infoLog = iota warnLog errorLog fatalLog traceLog + debugLog maxLevelLog ) var ( levels = map[int]string{ - debugLog: Color(Green, "[DEBUG]"), - infoLog: Color(Green, "[INFO ]"), - warnLog: Color(Yellow, "[WARN ]"), + infoLog: Color(White, "[INFO]"), + warnLog: Color(Yellow, "[WARN]"), errorLog: Color(Red, "[ERROR]"), - fatalLog: Color(Red, "[FATAL]"), - traceLog: Color(Pink, "[TRACE]"), + fatalLog: Color(Pink, "[FATAL]"), + traceLog: Color(Cyan, "[TRACE]"), + debugLog: Color(Green, "[DEBUG]"), } - Stdout = os.Stdout ) const ( - OutputPath = "./SPVLogs/" // The log files output path namePrefix = "LEVEL" callDepth = 2 KB_SIZE = int64(1024) @@ -74,8 +73,6 @@ func GetGID() uint64 { return n } -var Log *Logger - func LevelName(level int) string { if name, ok := levels[level]; ok { return name @@ -89,13 +86,17 @@ type Logger struct { maxLogsSize int64 // The max logs total size // Current log file and printer - printLock *sync.Mutex + mutex sync.Mutex maxPerLogSize int64 file *os.File logger *log.Logger watcher *fsnotify.Watcher } +func (l *Logger) Level() string { + return LevelName(l.level) +} + func (l *Logger) init() { // setup file watcher for the printing log file, // watch the file size change and trigger new log @@ -178,14 +179,14 @@ func (l *Logger) handleFileEvents(event fsnotify.Event) { case fsnotify.Write: info, _ := l.file.Stat() if info.Size() >= l.maxPerLogSize { - l.printLock.Lock() + l.mutex.Lock() // close previous log file l.file.Close() // unwatch it l.watcher.Remove(l.path + info.Name()) // create a new log file l.newLogFile() - l.printLock.Unlock() + l.mutex.Unlock() } } } @@ -194,7 +195,6 @@ func NewLogger(path string, level int, maxPerLogSizeMb, maxLogsSizeMb int64) *Lo logger := new(Logger) logger.path = path logger.level = level - logger.printLock = new(sync.Mutex) if maxPerLogSizeMb != 0 { logger.maxPerLogSize = maxPerLogSizeMb * MB_SIZE @@ -234,31 +234,15 @@ func newLogFile(path string) (*os.File, error) { return file, nil } -func Init(level int, maxPerLogSizeMb, maxLogsSizeMb int64) { - Log = NewLogger(OutputPath, level, maxPerLogSizeMb, maxLogsSizeMb) -} - func SortLogFiles(files []os.FileInfo) { sort.Sort(byTime(files)) } type byTime []os.FileInfo -// Len is the number of elements in the collection. -func (f byTime) Len() int { - return len(f) -} - -// Less reports whether the element with -// index i should sort before the element with index j. -func (f byTime) Less(i, j int) bool { - return f[i].Name() < f[j].Name() -} - -// Swap swaps the elements with indexes i and j. -func (f byTime) Swap(i, j int) { - f[i], f[j] = f[j], f[i] -} +func (f byTime) Len() int { return len(f) } +func (f byTime) Less(i, j int) bool { return f[i].Name() < f[j].Name() } +func (f byTime) Swap(i, j int) { f[i], f[j] = f[j], f[i] } func (l *Logger) SetPrintLevel(level int) error { if level > maxLevelLog || level < 0 { @@ -270,9 +254,9 @@ func (l *Logger) SetPrintLevel(level int) error { } func (l *Logger) Output(level int, a ...interface{}) error { - l.printLock.Lock() - defer l.printLock.Unlock() - if level >= l.level { + l.mutex.Lock() + defer l.mutex.Unlock() + if l.level >= level { gidStr := strconv.FormatUint(GetGID(), 10) a = append([]interface{}{LevelName(level), "GID", gidStr + ","}, a...) return l.logger.Output(callDepth, fmt.Sprintln(a...)) @@ -281,9 +265,9 @@ func (l *Logger) Output(level int, a ...interface{}) error { } func (l *Logger) Outputf(level int, format string, v ...interface{}) error { - l.printLock.Lock() - defer l.printLock.Unlock() - if level >= l.level { + l.mutex.Lock() + defer l.mutex.Unlock() + if l.level >= level { v = append([]interface{}{LevelName(level), "GID", GetGID()}, v...) return l.logger.Output(callDepth, fmt.Sprintf("%s %s %d, "+format+"\n", v...)) } @@ -291,55 +275,7 @@ func (l *Logger) Outputf(level int, format string, v ...interface{}) error { } func (l *Logger) Trace(a ...interface{}) { - l.Output(traceLog, a...) -} - -func (l *Logger) Tracef(format string, a ...interface{}) { - l.Outputf(traceLog, format, a...) -} - -func (l *Logger) Debug(a ...interface{}) { - l.Output(debugLog, a...) -} - -func (l *Logger) Debugf(format string, a ...interface{}) { - l.Outputf(debugLog, format, a...) -} - -func (l *Logger) Info(a ...interface{}) { - l.Output(infoLog, a...) -} - -func (l *Logger) Infof(format string, a ...interface{}) { - l.Outputf(infoLog, format, a...) -} - -func (l *Logger) Warn(a ...interface{}) { - l.Output(warnLog, a...) -} - -func (l *Logger) Warnf(format string, a ...interface{}) { - l.Outputf(warnLog, format, a...) -} - -func (l *Logger) Error(a ...interface{}) { - l.Output(errorLog, a...) -} - -func (l *Logger) Errorf(format string, a ...interface{}) { - l.Outputf(errorLog, format, a...) -} - -func (l *Logger) Fatal(a ...interface{}) { - l.Output(fatalLog, a...) -} - -func (l *Logger) Fatalf(format string, a ...interface{}) { - l.Outputf(fatalLog, format, a...) -} - -func Trace(a ...interface{}) { - if traceLog < Log.level { + if l.level < traceLog { return } @@ -355,11 +291,11 @@ func Trace(a ...interface{}) { a = append([]interface{}{funcName + "()", fileName + ":" + strconv.Itoa(line)}, a...) - Log.Trace(a...) + l.Output(traceLog, a...) } -func Tracef(format string, a ...interface{}) { - if traceLog < Log.level { +func (l *Logger) Tracef(format string, a ...interface{}) { + if l.level < traceLog { return } @@ -375,11 +311,11 @@ func Tracef(format string, a ...interface{}) { a = append([]interface{}{funcName, fileName, line}, a...) - Log.Tracef("%s() %s:%d "+format, a...) + l.Outputf(traceLog, "%s() %s:%d "+format, a...) } -func Debug(a ...interface{}) { - if debugLog < Log.level { +func (l *Logger) Debug(a ...interface{}) { + if l.level < debugLog { return } @@ -391,11 +327,11 @@ func Debug(a ...interface{}) { a = append([]interface{}{f.Name(), fileName + ":" + strconv.Itoa(line)}, a...) - Log.Debug(a...) + l.Output(debugLog, a...) } -func Debugf(format string, a ...interface{}) { - if debugLog < Log.level { +func (l *Logger) Debugf(format string, a ...interface{}) { + if l.level < debugLog { return } @@ -407,37 +343,37 @@ func Debugf(format string, a ...interface{}) { a = append([]interface{}{f.Name(), fileName, line}, a...) - Log.Debugf("%s %s:%d "+format, a...) + l.Outputf(debugLog, "%s %s:%d "+format, a...) } -func Info(a ...interface{}) { - Log.Info(a...) +func (l *Logger) Info(a ...interface{}) { + l.Output(infoLog, a...) } -func Warn(a ...interface{}) { - Log.Warn(a...) +func (l *Logger) Infof(format string, a ...interface{}) { + l.Outputf(infoLog, format, a...) } -func Error(a ...interface{}) { - Log.Error(a...) +func (l *Logger) Warn(a ...interface{}) { + l.Output(warnLog, a...) } -func Fatal(a ...interface{}) { - Log.Fatal(a...) +func (l *Logger) Warnf(format string, a ...interface{}) { + l.Outputf(warnLog, format, a...) } -func Infof(format string, a ...interface{}) { - Log.Infof(format, a...) +func (l *Logger) Error(a ...interface{}) { + l.Output(errorLog, a...) } -func Warnf(format string, a ...interface{}) { - Log.Warnf(format, a...) +func (l *Logger) Errorf(format string, a ...interface{}) { + l.Outputf(errorLog, format, a...) } -func Errorf(format string, a ...interface{}) { - Log.Errorf(format, a...) +func (l *Logger) Fatal(a ...interface{}) { + l.Output(fatalLog, a...) } -func Fatalf(format string, a ...interface{}) { - Log.Fatalf(format, a...) +func (l *Logger) Fatalf(format string, a ...interface{}) { + l.Outputf(fatalLog, format, a...) } diff --git a/log/log_test.go b/log/log_test.go index 898a2f7..509780b 100644 --- a/log/log_test.go +++ b/log/log_test.go @@ -6,7 +6,7 @@ import ( ) func TestNewLogger(t *testing.T) { - logger := NewLogger(0, 5, 20) + logger := NewLogger("./log-test",0, 5, 20) start := time.Now() for { logger.Info("Print info log") From e1547cad1f2871fa48095a0ac19bf3e7f7a66f52 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 10 Sep 2018 17:31:17 +0800 Subject: [PATCH 11/73] start peer message queue, add stall clear method and other minor fix --- peer/peer.go | 67 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/peer/peer.go b/peer/peer.go index 1c9de2f..9679ecb 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -61,6 +61,11 @@ type outMsg struct { doneChan chan<- struct{} } +// stallClearMsg is used to clear current stalled messages. This is useful when +// we are synced to current but a getblocks message is stalled, we need to cancel +// it. +type stallClearMsg struct{} + type Peer struct { *peer.Peer cfg Config @@ -69,8 +74,8 @@ type Peer struct { prevGetBlocksBegin *common.Uint256 prevGetBlocksStop *common.Uint256 - stallControl chan p2p.Message - blockQueue chan p2p.Message + stallControl chan interface{} + blockQueue chan interface{} outputQueue chan outMsg queueQuit chan struct{} } @@ -79,14 +84,15 @@ func NewPeer(peer *peer.Peer, cfg *Config) *Peer { p := Peer{ Peer: peer, cfg: *cfg, - stallControl: make(chan p2p.Message, 1), - blockQueue: make(chan p2p.Message, 1), + stallControl: make(chan interface{}, 1), + blockQueue: make(chan interface{}, 1), outputQueue: make(chan outMsg, outputBufferSize), queueQuit: make(chan struct{}), } peer.AddMessageFunc(p.handleMessage) go p.stallHandler() + go p.queueHandler() go p.blockHandler() go func() { @@ -98,29 +104,34 @@ func NewPeer(peer *peer.Peer, cfg *Config) *Peer { return &p } -func (p *Peer) sendMessage(msg outMsg) { - p.stallControl <- msg.msg - p.SendMessage(msg.msg, msg.doneChan) +func (p *Peer) sendMessage(out outMsg) { + switch out.msg.(type) { + case *msg.GetBlocks, *msg.GetData: + p.stallControl <- out.msg + } + p.SendMessage(out.msg, out.doneChan) } func (p *Peer) handleMessage(peer *peer.Peer, message p2p.Message) { - // Notify stall control - p.stallControl <- message - switch m := message.(type) { case *msg.Inv: + p.stallControl <- message p.cfg.OnInv(p, m) case *msg.MerkleBlock: + p.stallControl <- message p.blockQueue <- m case *msg.Tx: + p.stallControl <- message p.blockQueue <- m case *msg.NotFound: + p.stallControl <- message p.cfg.OnNotFound(p, m) case *msg.Reject: + p.stallControl <- message p.cfg.OnReject(p, m) } } @@ -147,7 +158,7 @@ out: // update last active time lastActive = time.Now() - switch message := ctrMsg.(type) { + switch m := ctrMsg.(type) { case *msg.GetBlocks: // Add expected response pendingResponses[p2p.CmdInv] = struct{}{} @@ -158,21 +169,25 @@ out: case *msg.GetData: // Add expected responses - for _, iv := range message.InvList { + for _, iv := range m.InvList { pendingResponses[iv.Hash.String()] = struct{}{} } case *msg.MerkleBlock: // Remove received merkleblock from expected response map. - delete(pendingResponses, message.Header.(*core.Header).Hash().String()) + delete(pendingResponses, m.Header.(*core.Header).Hash().String()) case *msg.Tx: // Remove received transaction from expected response map. - delete(pendingResponses, message.Serializable.(*core.Transaction).Hash().String()) + delete(pendingResponses, m.Serializable.(*core.Transaction).Hash().String()) case *msg.NotFound: // NotFound should not received from sync peer p.Disconnect() + + case stallClearMsg: + // Clear pending responses. + pendingResponses = make(map[string]struct{}) } case <-stallTicker.C: @@ -234,7 +249,7 @@ func (p *Peer) blockHandler() { notifyBlock := func() { // Notify OnBlock. p.cfg.OnBlock(p, &util.Block{ - Header: header, + Header: *header, Transactions: txs, }) @@ -267,21 +282,21 @@ out: continue } - // No transaction included in this block, so just notify block - // downloading completed. - if len(txIds) == 0 { - notifyBlock() - continue - } - // Set current downloading block header = &util.Header{ - Header: m.Header.(*core.Header), + Header: *m.Header.(*core.Header), NumTxs: m.Transactions, Hashes: m.Hashes, Flags: m.Flags, } + // No transaction included in this block, so just notify block + // downloading completed. + if len(txIds) == 0 { + notifyBlock() + continue + } + // Save pending transactions to cache. pendingTxs = make(map[common.Uint256]struct{}) for _, txId := range txIds { @@ -311,7 +326,7 @@ out: // Save downloaded transaction to cache. txs = append(txs, &util.Tx{ - Transaction: tx, + Transaction: *tx, Height: header.Height, }) @@ -457,6 +472,10 @@ func (p *Peer) PushGetBlocksMsg(locator []*common.Uint256, stopHash *common.Uint return nil } +func (p *Peer) StallClear() { + p.stallControl <- stallClearMsg{} +} + func (p *Peer) QueueMessage(msg p2p.Message, doneChan chan<- struct{}) { // Avoid risk of deadlock if goroutine already exited. The goroutine // we will be sending to hangs around until it knows for a fact that From ca789b16f0d1747e207c751172b88122344615ce Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 10 Sep 2018 17:34:43 +0800 Subject: [PATCH 12/73] add log.go to sdk package, minor fix in spvservice.go --- sdk/log.go | 28 +++++++++++++++++ sdk/spvservice.go | 76 ++++++++++++++++++++++++----------------------- 2 files changed, 67 insertions(+), 37 deletions(-) create mode 100644 sdk/log.go diff --git a/sdk/log.go b/sdk/log.go new file mode 100644 index 0000000..cec649f --- /dev/null +++ b/sdk/log.go @@ -0,0 +1,28 @@ +package sdk + +import ( + "github.com/elastos/Elastos.ELA.Utility/elalog" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log elalog.Logger + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = elalog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using elalog. +func UseLogger(logger elalog.Logger) { + log = logger +} diff --git a/sdk/spvservice.go b/sdk/spvservice.go index e9b4cb7..e932b38 100644 --- a/sdk/spvservice.go +++ b/sdk/spvservice.go @@ -5,7 +5,6 @@ import ( "time" "github.com/elastos/Elastos.ELA.SPV/blockchain" - "github.com/elastos/Elastos.ELA.SPV/log" spvpeer "github.com/elastos/Elastos.ELA.SPV/peer" "github.com/elastos/Elastos.ELA.SPV/sync" "github.com/elastos/Elastos.ELA.SPV/util" @@ -47,10 +46,10 @@ type service struct { cfg Config syncManager *sync.SyncManager - newPeerQueue chan *peer.Peer - donePeerQueue chan *peer.Peer - txQueue chan interface{} - quit chan struct{} + newPeers chan *peer.Peer + donePeers chan *peer.Peer + txQueue chan interface{} + quit chan struct{} } // Create a instance of SPV service implementation. @@ -63,11 +62,11 @@ func NewSPVService(cfg *Config) (*service, error) { // Create SPV service instance service := &service{ - cfg: *cfg, - newPeerQueue: make(chan *peer.Peer), - donePeerQueue: make(chan *peer.Peer), - txQueue: make(chan interface{}, 3), - quit: make(chan struct{}), + cfg: *cfg, + newPeers: make(chan *peer.Peer, cfg.MaxPeers), + donePeers: make(chan *peer.Peer, cfg.MaxPeers), + txQueue: make(chan interface{}, 3), + quit: make(chan struct{}), } var maxPeers int @@ -103,6 +102,8 @@ func NewSPVService(cfg *Config) (*service, error) { func() uint64 { return uint64(chain.BestHeight()) }, ) serverCfg.MaxPeers = maxPeers + serverCfg.DisableListen = true + serverCfg.DisableRelayTx = true // Create P2P server. server, err := server.NewServer(serverCfg) @@ -115,7 +116,7 @@ func NewSPVService(cfg *Config) (*service, error) { } func (s *service) start() { - go s.peersHandler() + go s.peerHandler() go s.txHandler() } @@ -169,22 +170,24 @@ func (s *service) makeEmptyMessage(cmd string) (p2p.Message, error) { } func (s *service) newPeer(peer *peer.Peer) { - s.newPeerQueue <- peer + log.Debugf("server new peer %v", peer) + s.newPeers <- peer } func (s *service) donePeer(peer *peer.Peer) { - s.donePeerQueue <- peer + log.Debugf("server done peer %v", peer) + s.donePeers <- peer } -// peersHandler handles new peers and done peers from P2P server. +// peerHandler handles new peers and done peers from P2P server. // When comes new peer, create a spv peer warpper for it -func (s *service) peersHandler() { +func (s *service) peerHandler() { peers := make(map[*peer.Peer]*spvpeer.Peer) out: for { select { - case p := <-s.newPeerQueue: + case p := <-s.newPeers: // Create spv peer warpper for the new peer. sp := spvpeer.NewPeer(p, &spvpeer.Config{ @@ -198,7 +201,7 @@ out: peers[p] = sp s.syncManager.NewPeer(sp) - case p := <-s.donePeerQueue: + case p := <-s.donePeers: sp, ok := peers[p] if !ok { log.Errorf("unknown done peer %v", p) @@ -217,8 +220,8 @@ out: cleanup: for { select { - case <-s.newPeerQueue: - case <-s.donePeerQueue: + case <-s.newPeers: + case <-s.donePeers: default: break cleanup } @@ -229,7 +232,7 @@ cleanup: // txHandler handles transaction messages like send transaction, transaction inv // transaction reject etc. func (s *service) txHandler() { - var unconfirmed = make(map[common.Uint256]sendTxMsg) + var unconfirmed = make(map[common.Uint256]*sendTxMsg) var accepted = make(map[common.Uint256]struct{}) var rejected = make(map[common.Uint256]struct{}) @@ -241,7 +244,7 @@ out: select { case tmsg := <-s.txQueue: switch tmsg := tmsg.(type) { - case sendTxMsg: + case *sendTxMsg: txId := tmsg.tx.Hash() tmsg.expire = time.Now().Add(TxExpireTime) unconfirmed[txId] = tmsg @@ -251,7 +254,7 @@ out: // Broadcast unconfirmed transaction s.IServer.BroadcastMessage(msg.NewTx(tmsg.tx)) - case txInvMsg: + case *txInvMsg: // When a transaction was accepted and add to the txMemPool, a // txInv message will be received through message relay, but it // only works when there are more than 2 peers connected. @@ -272,14 +275,14 @@ out: if s.cfg.StateNotifier != nil { s.cfg.StateNotifier.TransactionAccepted( &util.Tx{ - Transaction: tx, + Transaction: *tx, Height: 0, }) } }(txMsg.tx) } - case txRejectMsg: + case *txRejectMsg: // If some of the peers are bad actors, transaction can be both // accepted and rejected. For we can not say who are bad actors // and who are not, so just pick the first response and notify @@ -301,38 +304,37 @@ out: if s.cfg.StateNotifier != nil { s.cfg.StateNotifier.TransactionRejected( &util.Tx{ - Transaction: tx, + Transaction: *tx, Height: 0, }) } }(txMsg.tx) } - case blockMsg: + case *blockMsg: // Loop through all packed transactions, see if match to any // sent transactions. - confirmedTxs := make(map[util.Tx]struct{}) + confirmedTxs := make(map[common.Uint256]*util.Tx) for _, tx := range tmsg.block.Transactions { txId := tx.Hash() tx.Height = tmsg.block.Height if _, ok := unconfirmed[txId]; ok { - confirmedTxs[*tx] = struct{}{} + confirmedTxs[txId] = tx continue } if _, ok := accepted[txId]; ok { - confirmedTxs[*tx] = struct{}{} + confirmedTxs[txId] = tx continue } if _, ok := rejected[txId]; ok { - confirmedTxs[*tx] = struct{}{} + confirmedTxs[txId] = tx } } - for tx := range confirmedTxs { - txId := tx.Hash() + for txId, tx := range confirmedTxs { delete(unconfirmed, txId) delete(accepted, txId) delete(rejected, txId) @@ -342,7 +344,7 @@ out: if s.cfg.StateNotifier != nil { s.cfg.StateNotifier.TransactionConfirmed(tx) } - }(&tx) + }(tx) } } case <-retryTicker.C: @@ -387,7 +389,7 @@ func (s *service) SendTransaction(tx core.Transaction) error { return fmt.Errorf("spv service did not sync to current") } - s.txQueue <- sendTxMsg{tx: &tx} + s.txQueue <- &sendTxMsg{tx: &tx} return nil } @@ -398,7 +400,7 @@ func (s *service) onInv(sp *spvpeer.Peer, inv *msg.Inv) { for _, iv := range inv.InvList { switch iv.Type { case msg.InvTypeTx: - s.txQueue <- txInvMsg{iv: iv} + s.txQueue <- &txInvMsg{iv: iv} } } } @@ -413,7 +415,7 @@ func (s *service) onBlock(sp *spvpeer.Peer, block *util.Block) { go func() { select { case <-done: - s.txQueue <- blockMsg{block: block} + s.txQueue <- &blockMsg{block: block} if s.cfg.StateNotifier != nil { s.cfg.StateNotifier.BlockCommitted(block) } @@ -439,7 +441,7 @@ func (s *service) onNotFound(sp *spvpeer.Peer, notFound *msg.NotFound) { func (s *service) onReject(sp *spvpeer.Peer, reject *msg.Reject) { if reject.Cmd == p2p.CmdTx { - s.txQueue <- txInvMsg{iv: &msg.InvVect{Type: msg.InvTypeTx, Hash: reject.Hash}} + s.txQueue <- &txRejectMsg{iv: &msg.InvVect{Type: msg.InvTypeTx, Hash: reject.Hash}} } log.Warnf("reject message from peer %v: Code: %s, Hash %s, Reason: %s", sp, reject.Code.String(), reject.Hash.String(), reject.Reason) From 97b08210eb6518ef00ab8828870f8991d02f84fb Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 10 Sep 2018 17:35:48 +0800 Subject: [PATCH 13/73] minor fix in blockchain/chain.go --- blockchain/chain.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/blockchain/chain.go b/blockchain/chain.go index 016b486..24d1386 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -46,7 +46,7 @@ func New(foundation string, db database.ChainStore) (*BlockChain, error) { return nil, errors.New("parse foundation address failed") } genesisHeader := GenesisHeader(foundationAddress) - storeHeader := &util.Header{Header: genesisHeader, TotalWork: new(big.Int)} + storeHeader := &util.Header{Header: *genesisHeader, TotalWork: new(big.Int)} chain.db.Headers().Put(storeHeader, true) } @@ -54,7 +54,7 @@ func New(foundation string, db database.ChainStore) (*BlockChain, error) { } func (b *BlockChain) CommitTx(tx *core.Transaction) (bool, error) { - return b.db.StoreTx(&util.Tx{Transaction: tx, Height: 0}) + return b.db.StoreTx(&util.Tx{Transaction: *tx, Height: 0}) } func (b *BlockChain) CommitBlock(block *util.Block) (newTip, reorg bool, newHeight, fps uint32, err error) { @@ -62,7 +62,7 @@ func (b *BlockChain) CommitBlock(block *util.Block) (newTip, reorg bool, newHeig defer b.lock.Unlock() newTip = false reorg = false - var header = block.Header + var header = &block.Header var commonAncestor *util.Header // Fetch our current best header from the db bestHeader, err := b.db.Headers().GetBest() @@ -77,7 +77,7 @@ func (b *BlockChain) CommitBlock(block *util.Block) (newTip, reorg bool, newHeig if block.Previous.IsEqual(tipHash) { parentHeader = bestHeader } else { - parentHeader, err = b.db.Headers().GetPrevious(block.Header) + parentHeader, err = b.db.Headers().GetPrevious(header) if err != nil { return false, false, 0, 0, OrphanBlockError } From 6e4b6bdf2fd8613c1b462b27b3b5eaacb1f7344b Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 10 Sep 2018 17:37:34 +0800 Subject: [PATCH 14/73] minor fix in database package --- database/defaultdb.go | 2 +- database/headersonly.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/database/defaultdb.go b/database/defaultdb.go index a3410f5..ccf8e6e 100644 --- a/database/defaultdb.go +++ b/database/defaultdb.go @@ -16,7 +16,7 @@ func (d *defaultChainDB) Headers() Headers { // StoreBlock save a block into database, returns how many // false positive transactions are and error. func (d *defaultChainDB) StoreBlock(block *util.Block, newTip bool) (fps uint32, err error) { - err = d.h.Put(block.Header, newTip) + err = d.h.Put(&block.Header, newTip) if err != nil { return 0, err } diff --git a/database/headersonly.go b/database/headersonly.go index cd9068d..bacc34a 100644 --- a/database/headersonly.go +++ b/database/headersonly.go @@ -17,7 +17,7 @@ func (h *headersOnlyChainDB) Headers() Headers { // StoreBlock save a block into database, returns how many // false positive transactions are and error. func (h *headersOnlyChainDB) StoreBlock(block *util.Block, newTip bool) (fps uint32, err error) { - return fps, h.db.Put(block.Header, newTip) + return fps, h.db.Put(&block.Header, newTip) } // StoreTx save a transaction into database, and return From 5b771e67a6d2a9f33e7abc560ed7410eae8cd776 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 10 Sep 2018 20:25:00 +0800 Subject: [PATCH 15/73] minor fix in util package --- util/block.go | 4 ++-- util/header.go | 2 +- util/tx.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util/block.go b/util/block.go index 704c2b0..650a931 100644 --- a/util/block.go +++ b/util/block.go @@ -3,8 +3,8 @@ package util // Block represent a block that stored into // blockchain database. type Block struct { - // The store header of this block. - *Header + // header of this block. + Header // Transactions of this block. Transactions []*Tx diff --git a/util/header.go b/util/header.go index bb0dca9..ffbec39 100644 --- a/util/header.go +++ b/util/header.go @@ -11,7 +11,7 @@ import ( // Header is a data structure stored in database. type Header struct { // The origin header of the block - *core.Header + core.Header // MerkleProof for transactions packed in this block NumTxs uint32 diff --git a/util/tx.go b/util/tx.go index 765180f..94e11e6 100644 --- a/util/tx.go +++ b/util/tx.go @@ -5,7 +5,7 @@ import "github.com/elastos/Elastos.ELA/core" // Tx is a data structure used in database. type Tx struct { // The origin transaction data. - *core.Transaction + core.Transaction // The block height that this transaction // belongs to. From f86340e9ec51b59514b0b282de14d3f12671396b Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 11 Sep 2018 10:36:47 +0800 Subject: [PATCH 16/73] add DefaultPort into spv service configuration --- sdk/interface.go | 3 +++ sdk/spvservice.go | 1 + 2 files changed, 4 insertions(+) diff --git a/sdk/interface.go b/sdk/interface.go index db1eef1..a6a3683 100644 --- a/sdk/interface.go +++ b/sdk/interface.go @@ -59,6 +59,9 @@ type Config struct { // The seed peers addresses in [host:port] or [ip:port] format. SeedList []string + // The default port for public peers to provide service. + DefaultPort uint16 + // The max peer connections. MaxPeers int diff --git a/sdk/spvservice.go b/sdk/spvservice.go index e932b38..198ecad 100644 --- a/sdk/spvservice.go +++ b/sdk/spvservice.go @@ -94,6 +94,7 @@ func NewSPVService(cfg *Config) (*service, error) { // Initiate P2P server configuration serverCfg := server.NewDefaultConfig( cfg.Magic, + cfg.DefaultPort, cfg.SeedList, nil, service.newPeer, From 14dbfdb9bb6b101392dabd41de1cbb371c06d087 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 11 Sep 2018 10:38:40 +0800 Subject: [PATCH 17/73] modify syncmanager sync progress logic to get better performance --- sync/manager.go | 77 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/sync/manager.go b/sync/manager.go index e8c6690..952b88c 100644 --- a/sync/manager.go +++ b/sync/manager.go @@ -1,7 +1,6 @@ package sync import ( - "sync" "sync/atomic" "github.com/elastos/Elastos.ELA.SPV/blockchain" @@ -15,12 +14,20 @@ import ( ) const ( + // minPendingRequests is the minimum number of block hashes in request + // queue. + minPendingRequests = msg.MaxInvPerMsg + + // minInFlightBlocks is the minimum number of blocks that should be + // in the request queue before requesting more. + minInFlightBlocks = 10 + // maxBadBlockRate is the maximum bad blocks rate of received blocks. maxBadBlockRate float64 = 0.001 // maxFalsePositiveRate is the maximum false positive rate of received // transactions. - maxFalsePositiveRate float64 = 0.001 + maxFalsePositiveRate float64 = 0.0001 // maxRequestedBlocks is the maximum number of requested block // hashes to store in memory. @@ -119,7 +126,6 @@ type SyncManager struct { shutdown int32 cfg Config msgChan chan interface{} - wg sync.WaitGroup quit chan struct{} // These fields should only be accessed from the blockHandler thread @@ -215,6 +221,9 @@ func (sm *SyncManager) syncWith(p *peer.Peer) { // isSyncCandidate returns whether or not the peer is a candidate to consider // syncing from. func (sm *SyncManager) isSyncCandidate(peer *peer.Peer) bool { + // Just return true. + return true + services := peer.Services() // Candidate if all checks passed. return services&p2p.SFNodeNetwork == p2p.SFNodeNetwork && @@ -236,6 +245,7 @@ func (sm *SyncManager) getSyncCandidates() []*peer.Peer { // updateBloomFilter update the bloom filter and send it to the given peer. func (sm *SyncManager) updateBloomFilter(p *peer.Peer) { msg := sm.cfg.UpdateFilter().GetFilterLoadMsg() + log.Debugf("Update bloom filter %v, %d, %d", msg.Filter, msg.Tweak, msg.HashFuncs) doneChan := make(chan struct{}) p.QueueMessage(msg, doneChan) @@ -274,6 +284,8 @@ func (sm *SyncManager) handleNewPeerMsg(peer *peer.Peer) { requestedBlocks: make(map[common.Uint256]struct{}), } + sm.updateBloomFilter(peer) + // Start syncing by choosing the best candidate if needed. if isSyncCandidate && sm.syncPeer == nil { sm.startSync() @@ -354,7 +366,6 @@ func (sm *SyncManager) handleTxMsg(tmsg *txMsg) { sm.updateBloomFilter(peer) } } - } // handleBlockMsg handles block messages from all peers. Blocks are requested @@ -429,7 +440,7 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { // Check false positive rate. state.falsePositives += fps if state.falsePosRate() > maxFalsePositiveRate { - + sm.updateBloomFilter(peer) } // We can exit here if the block is already known @@ -453,21 +464,19 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { // If we're current now, nothing more to do. if sm.current() { + // When we are current, the last getblocks message we sent will get + // stalled, so we cancel it to prevent peer from stall disconnection. + peer.StallClear() peer.UpdateHeight(newHeight) return } - // If we're not current and we've downloaded everything we've requested send another getblocks message. - // Otherwise we'll request the next block in the queue. - if len(state.requestQueue) == 0 { - locator := sm.cfg.Chain.LatestBlockLocator() - peer.PushGetBlocksMsg(locator, &zeroHash) - log.Debug("Request queue at zero. Pushing new locator.") - return + // Request more blocks if in flight blocks is getting short. This can make + // syncing progress a little bit faster then request more blocks after the + // last requested block received. + if len(state.requestedBlocks) < minInFlightBlocks { + sm.requestQueuedInv(peer, state) } - - // We have pending requests, so push a new getdata message. - sm.pushGetDataMsg(peer, state) } // haveInventory returns whether or not the inventory represented by the passed @@ -505,7 +514,16 @@ func (sm *SyncManager) handleInvMsg(imsg *invMsg) { return } + // Attempt to find the final block in the inventory list. There may + // not be one. + var lastBlock *msg.InvVect invVects := imsg.inv.InvList + for i := len(invVects) - 1; i >= 0; i-- { + if invVects[i].Type == msg.InvTypeBlock { + lastBlock = invVects[i] + break + } + } // Ignore invs from peers that aren't the sync if we are not current. // Helps prevent fetching a mass of orphans. @@ -518,7 +536,6 @@ func (sm *SyncManager) handleInvMsg(imsg *invMsg) { // Ignore unsupported inventory types. switch iv.Type { case msg.InvTypeBlock: - iv.Type = msg.InvTypeFilteredBlock case msg.InvTypeTx: default: continue @@ -532,10 +549,19 @@ func (sm *SyncManager) handleInvMsg(imsg *invMsg) { } } - sm.pushGetDataMsg(peer, state) + // Check if we are in syncing mode and the request queue is not long enough. + if !sm.current() && len(state.requestQueue) < minPendingRequests { + if lastBlock != nil { + locator := []*common.Uint256{&lastBlock.Hash} + peer.PushGetBlocksMsg(locator, &zeroHash) + } + } + + // If there are any queued inventory, just request them. + sm.requestQueuedInv(peer, state) } -func (sm *SyncManager) pushGetDataMsg(peer *peer.Peer, state *peerSyncState) { +func (sm *SyncManager) requestQueuedInv(peer *peer.Peer, state *peerSyncState) { // Request as much as possible at once. Anything that won't fit into // the request will be requested on the next inv message. numRequested := 0 @@ -547,7 +573,7 @@ func (sm *SyncManager) pushGetDataMsg(peer *peer.Peer, state *peerSyncState) { requestQueue = requestQueue[1:] switch iv.Type { - case msg.InvTypeFilteredBlock: + case msg.InvTypeBlock: // Request the block if there is not already a pending // request. if _, exists := sm.requestedBlocks[iv.Hash]; !exists { @@ -555,6 +581,7 @@ func (sm *SyncManager) pushGetDataMsg(peer *peer.Peer, state *peerSyncState) { sm.limitMap(sm.requestedBlocks, maxRequestedBlocks) state.requestedBlocks[iv.Hash] = struct{}{} + iv.Type = msg.InvTypeFilteredBlock gdmsg.AddInvVect(iv) numRequested++ } @@ -578,6 +605,7 @@ func (sm *SyncManager) pushGetDataMsg(peer *peer.Peer, state *peerSyncState) { } state.requestQueue = requestQueue if len(gdmsg.InvList) > 0 { + log.Debugf("QueueMessage getdata size %d", len(gdmsg.InvList)) peer.QueueMessage(gdmsg, nil) } } @@ -653,7 +681,14 @@ out: } } - sm.wg.Done() +cleanup: + for { + select { + case <-sm.msgChan: + default: + break cleanup + } + } log.Trace("Block handler done") } @@ -721,7 +756,6 @@ func (sm *SyncManager) Start() { } log.Trace("Starting sync manager") - sm.wg.Add(1) go sm.blockHandler() } @@ -736,7 +770,6 @@ func (sm *SyncManager) Stop() error { log.Infof("Sync manager shutting down") close(sm.quit) - sm.wg.Wait() return nil } From 9008704e92b5b7917c2a11349f15355e7398e480 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 11 Sep 2018 10:39:16 +0800 Subject: [PATCH 18/73] initial refactor commit for spvwallet/client --- spvwallet/client/account/account.go | 26 +++++++---------- spvwallet/client/common.go | 6 ++-- spvwallet/client/database/database.go | 28 +++++++++--------- spvwallet/client/interface.go | 26 +++++++---------- spvwallet/client/keystore.go | 32 ++++++++++----------- spvwallet/client/keystore_file.go | 2 +- spvwallet/client/transaction/transaction.go | 28 +++++++++--------- spvwallet/client/wallet/wallet.go | 31 ++++++++++---------- 8 files changed, 84 insertions(+), 95 deletions(-) diff --git a/spvwallet/client/account/account.go b/spvwallet/client/account/account.go index 1da5b70..44ee0aa 100644 --- a/spvwallet/client/account/account.go +++ b/spvwallet/client/account/account.go @@ -7,8 +7,7 @@ import ( "os" "strings" - "github.com/elastos/Elastos.ELA.SPV/log" - . "github.com/elastos/Elastos.ELA.SPV/spvwallet/cli" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/client" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" @@ -19,19 +18,18 @@ const ( MinMultiSignKeys = 3 ) -func listBalanceInfo(wallet Wallet) error { +func listBalanceInfo(wallet client.Wallet) error { addrs, err := wallet.GetAddrs() if err != nil { - log.Error("Get addresses error:", err) return errors.New("get wallet addresses failed") } - return ShowAccounts(addrs, nil, wallet) + return client.ShowAccounts(addrs, nil, wallet) } -func newSubAccount(password []byte, wallet Wallet) error { +func newSubAccount(password []byte, wallet client.Wallet) error { var err error - password, err = GetPassword(password, false) + password, err = client.GetPassword(password, false) if err != nil { return err } @@ -43,14 +41,13 @@ func newSubAccount(password []byte, wallet Wallet) error { addrs, err := wallet.GetAddrs() if err != nil { - log.Error("Get addresses error:", err) return errors.New("get wallet addresses failed") } - return ShowAccounts(addrs, programHash, wallet) + return client.ShowAccounts(addrs, programHash, wallet) } -func addMultiSignAccount(context *cli.Context, wallet Wallet, content string) error { +func addMultiSignAccount(context *cli.Context, wallet client.Wallet, content string) error { // Get address content from file or cli input publicKeys, err := getPublicKeys(content) if err != nil { @@ -77,11 +74,10 @@ func addMultiSignAccount(context *cli.Context, wallet Wallet, content string) er addrs, err := wallet.GetAddrs() if err != nil { - log.Error("Get addresses error:", err) return errors.New("get wallet addresses failed") } - return ShowAccounts(addrs, programHash, wallet) + return client.ShowAccounts(addrs, programHash, wallet) } func getPublicKeys(content string) ([]*crypto.PublicKey, error) { @@ -141,7 +137,7 @@ func accountAction(context *cli.Context) { } pass := context.String("password") - wallet, err := Open() + wallet, err := client.Open() if err != nil { fmt.Println("error: open wallet failed,", err) os.Exit(2) @@ -149,7 +145,7 @@ func accountAction(context *cli.Context) { // list accounts if context.Bool("list") { - if err := ShowAccountInfo([]byte(pass)); err != nil { + if err := client.ShowAccountInfo([]byte(pass)); err != nil { fmt.Println("error: list accounts info failed,", err) cli.ShowCommandHelpAndExit(context, "list", 3) } @@ -191,7 +187,7 @@ func NewCommand() cli.Command { Usage: "account [command] [args]", Description: "commands to create new sub account or multisig account and show accounts balances", ArgsUsage: "[args]", - Flags: append(CommonFlags, + Flags: append(client.CommonFlags, cli.BoolFlag{ Name: "list, l", Usage: "list all accounts, including address, public key and type", diff --git a/spvwallet/client/common.go b/spvwallet/client/common.go index 75b00df..c33faa8 100644 --- a/spvwallet/client/common.go +++ b/spvwallet/client/common.go @@ -1,4 +1,4 @@ -package cli +package client import ( "bufio" @@ -8,7 +8,7 @@ import ( "strconv" "strings" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/howeyc/gopass" @@ -114,7 +114,7 @@ func SelectAccount(wallet Wallet) (string, error) { return addrs[index].String(), nil } -func ShowAccounts(addrs []*util.Addr, newAddr *common.Uint168, wallet Wallet) error { +func ShowAccounts(addrs []*sutil.Addr, newAddr *common.Uint168, wallet Wallet) error { // print header fmt.Printf("%5s %34s %-20s%22s %6s\n", "INDEX", "ADDRESS", "BALANCE", "(LOCKED)", "TYPE") fmt.Println("-----", strings.Repeat("-", 34), strings.Repeat("-", 42), "------") diff --git a/spvwallet/client/database/database.go b/spvwallet/client/database/database.go index 90d79ad..205843a 100644 --- a/spvwallet/client/database/database.go +++ b/spvwallet/client/database/database.go @@ -3,20 +3,20 @@ package database import ( "sync" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/store" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/headers" "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/sqlite" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" ) type Database interface { AddAddress(address *common.Uint168, script []byte, addrType int) error - GetAddress(address *common.Uint168) (*util.Addr, error) - GetAddrs() ([]*util.Addr, error) + GetAddress(address *common.Uint168) (*sutil.Addr, error) + GetAddrs() ([]*sutil.Addr, error) DeleteAddress(address *common.Uint168) error - GetAddressUTXOs(address *common.Uint168) ([]*util.UTXO, error) - GetAddressSTXOs(address *common.Uint168) ([]*util.STXO, error) + GetAddressUTXOs(address *common.Uint168) ([]*sutil.UTXO, error) + GetAddressSTXOs(address *common.Uint168) ([]*sutil.STXO, error) BestHeight() uint32 Clear() error } @@ -35,7 +35,7 @@ func New() (Database, error) { type database struct { lock *sync.RWMutex - store store.DataStore + store sqlite.DataStore } func (d *database) AddAddress(address *common.Uint168, script []byte, addrType int) error { @@ -45,14 +45,14 @@ func (d *database) AddAddress(address *common.Uint168, script []byte, addrType i return d.store.Addrs().Put(address, script, addrType) } -func (d *database) GetAddress(address *common.Uint168) (*util.Addr, error) { +func (d *database) GetAddress(address *common.Uint168) (*sutil.Addr, error) { d.lock.RLock() defer d.lock.RUnlock() return d.store.Addrs().Get(address) } -func (d *database) GetAddrs() ([]*util.Addr, error) { +func (d *database) GetAddrs() ([]*sutil.Addr, error) { d.lock.RLock() defer d.lock.RUnlock() @@ -63,17 +63,17 @@ func (d *database) DeleteAddress(address *common.Uint168) error { d.lock.Lock() defer d.lock.Unlock() - return d.store.Addrs().Delete(address) + return d.store.Addrs().Del(address) } -func (d *database) GetAddressUTXOs(address *common.Uint168) ([]*util.UTXO, error) { +func (d *database) GetAddressUTXOs(address *common.Uint168) ([]*sutil.UTXO, error) { d.lock.RLock() defer d.lock.RUnlock() return d.store.UTXOs().GetAddrAll(address) } -func (d *database) GetAddressSTXOs(address *common.Uint168) ([]*util.STXO, error) { +func (d *database) GetAddressSTXOs(address *common.Uint168) ([]*sutil.STXO, error) { d.lock.RLock() defer d.lock.RUnlock() @@ -84,14 +84,14 @@ func (d *database) BestHeight() uint32 { d.lock.RLock() defer d.lock.RUnlock() - return d.store.Chain().GetHeight() + return d.store.State().GetHeight() } func (d *database) Clear() error { d.lock.Lock() defer d.lock.Unlock() - headers, err := database.NewHeadersDB() + headers, err := headers.New() if err != nil { return err } diff --git a/spvwallet/client/interface.go b/spvwallet/client/interface.go index cc83967..eaa8b14 100644 --- a/spvwallet/client/interface.go +++ b/spvwallet/client/interface.go @@ -1,4 +1,4 @@ -package cli +package client import ( "bytes" @@ -7,11 +7,10 @@ import ( "math/rand" "strconv" - "github.com/elastos/Elastos.ELA.SPV/log" "github.com/elastos/Elastos.ELA.SPV/sdk" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/cli/database" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/client/database" "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" @@ -50,25 +49,22 @@ type wallet struct { func Create(password []byte) error { keyStore, err := CreateKeystore(password) if err != nil { - log.Error("Wallet create keystore failed:", err) return err } database, err := database.New() if err != nil { - log.Error("Wallet create database failed:", err) return err } mainAccount := keyStore.GetAccountByIndex(0) return database.AddAddress(mainAccount.ProgramHash(), - mainAccount.RedeemScript(), util.TypeMaster) + mainAccount.RedeemScript(), sutil.TypeMaster) } func Open() (Wallet, error) { database, err := database.New() if err != nil { - log.Error("Wallet open database failed:", err) return nil, err } @@ -93,7 +89,7 @@ func (wallet *wallet) NewSubAccount(password []byte) (*common.Uint168, error) { } account := wallet.Keystore.NewAccount() - err = wallet.AddAddress(account.ProgramHash(), account.RedeemScript(), util.TypeSub) + err = wallet.AddAddress(account.ProgramHash(), account.RedeemScript(), sutil.TypeSub) if err != nil { return nil, err } @@ -115,7 +111,7 @@ func (wallet *wallet) AddMultiSignAccount(M uint, publicKeys ...*crypto.PublicKe return nil, errors.New("[Wallet], CreateMultiSignAddress failed") } - err = wallet.AddAddress(programHash, redeemScript, util.TypeMulti) + err = wallet.AddAddress(programHash, redeemScript, sutil.TypeMulti) if err != nil { return nil, err } @@ -177,8 +173,8 @@ func (wallet *wallet) createTransaction(fromAddress string, fee *common.Fixed64, if err != nil { return nil, errors.New("[Wallet], Get spender's UTXOs failed") } - availableUTXOs := wallet.removeLockedUTXOs(utxos) // Remove locked UTXOs - availableUTXOs = util.SortUTXOs(availableUTXOs) // Sort available UTXOs by value ASC + availableUTXOs := wallet.removeLockedUTXOs(utxos) // Remove locked UTXOs + availableUTXOs = sutil.SortByValue(availableUTXOs) // Sort available UTXOs by value ASC // Create transaction inputs var txInputs []*core.Input // The inputs in transaction @@ -334,8 +330,8 @@ func getSystemAssetId() common.Uint256 { return systemToken.Hash() } -func (wallet *wallet) removeLockedUTXOs(utxos []*util.UTXO) []*util.UTXO { - var availableUTXOs []*util.UTXO +func (wallet *wallet) removeLockedUTXOs(utxos []*sutil.UTXO) []*sutil.UTXO { + var availableUTXOs []*sutil.UTXO var currentHeight = wallet.BestHeight() for _, utxo := range utxos { if utxo.AtHeight == 0 { // remove unconfirmed UTOXs @@ -352,7 +348,7 @@ func (wallet *wallet) removeLockedUTXOs(utxos []*util.UTXO) []*util.UTXO { return availableUTXOs } -func InputFromUTXO(utxo *util.UTXO) *core.Input { +func InputFromUTXO(utxo *sutil.UTXO) *core.Input { input := new(core.Input) input.Previous.TxID = utxo.Op.TxID input.Previous.Index = utxo.Op.Index diff --git a/spvwallet/client/keystore.go b/spvwallet/client/keystore.go index 0d01d26..bc4ef9a 100644 --- a/spvwallet/client/keystore.go +++ b/spvwallet/client/keystore.go @@ -1,4 +1,4 @@ -package spvwallet +package client import ( "crypto/rand" @@ -7,7 +7,7 @@ import ( "fmt" "sync" - . "github.com/elastos/Elastos.ELA.SPV/sdk" + "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" ) @@ -19,11 +19,11 @@ const ( type Keystore interface { ChangePassword(old, new []byte) error - MainAccount() *Account - NewAccount() *Account - GetAccounts() []*Account - GetAccountByIndex(index int) *Account - GetAccountByProgramHash(programHash *common.Uint168) *Account + MainAccount() *sdk.Account + NewAccount() *sdk.Account + GetAccounts() []*sdk.Account + GetAccountByIndex(index int) *sdk.Account + GetAccountByProgramHash(programHash *common.Uint168) *sdk.Account Json() (string, error) FromJson(json string, password string) error @@ -36,7 +36,7 @@ type KeystoreImpl struct { masterKey []byte - accounts []*Account + accounts []*sdk.Account } func CreateKeystore(password []byte) (Keystore, error) { @@ -130,7 +130,7 @@ func (store *KeystoreImpl) initKeystore(keystoreFile *KeystoreFile, password []b func (store *KeystoreImpl) initAccounts(masterKey, privateKey []byte, publicKey *crypto.PublicKey) error { // initiate main account - mainAccount, err := NewAccount(privateKey, publicKey) + mainAccount, err := sdk.NewAccount(privateKey, publicKey) if err != nil { return err } @@ -147,7 +147,7 @@ func (store *KeystoreImpl) initAccounts(masterKey, privateKey []byte, publicKey if err != nil { return err } - childAccount, err := NewAccount(privateKey, publicKey) + childAccount, err := sdk.NewAccount(privateKey, publicKey) if err != nil { return err } @@ -218,11 +218,11 @@ func (store *KeystoreImpl) ChangePassword(oldPassword, newPassword []byte) error return nil } -func (store *KeystoreImpl) MainAccount() *Account { +func (store *KeystoreImpl) MainAccount() *sdk.Account { return store.GetAccountByIndex(0) } -func (store *KeystoreImpl) NewAccount() *Account { +func (store *KeystoreImpl) NewAccount() *sdk.Account { // create sub account privateKey, publicKey, err := crypto.GenerateSubKeyPair( store.SubAccountsCount+1, store.masterKey, store.accounts[0].PrivateKey()) @@ -230,7 +230,7 @@ func (store *KeystoreImpl) NewAccount() *Account { panic(fmt.Sprint("New sub account failed,", err)) } - account, err := NewAccount(privateKey, publicKey) + account, err := sdk.NewAccount(privateKey, publicKey) if err != nil { panic(fmt.Sprint("New sub account failed,", err)) } @@ -246,18 +246,18 @@ func (store *KeystoreImpl) NewAccount() *Account { return account } -func (store *KeystoreImpl) GetAccounts() []*Account { +func (store *KeystoreImpl) GetAccounts() []*sdk.Account { return store.accounts } -func (store *KeystoreImpl) GetAccountByIndex(index int) *Account { +func (store *KeystoreImpl) GetAccountByIndex(index int) *sdk.Account { if index < 0 || index > len(store.accounts)-1 { return nil } return store.accounts[index] } -func (store *KeystoreImpl) GetAccountByProgramHash(programHash *common.Uint168) *Account { +func (store *KeystoreImpl) GetAccountByProgramHash(programHash *common.Uint168) *sdk.Account { if programHash == nil { return nil } diff --git a/spvwallet/client/keystore_file.go b/spvwallet/client/keystore_file.go index 98a829d..cd9d033 100644 --- a/spvwallet/client/keystore_file.go +++ b/spvwallet/client/keystore_file.go @@ -1,4 +1,4 @@ -package spvwallet +package client import ( "encoding/json" diff --git a/spvwallet/client/transaction/transaction.go b/spvwallet/client/transaction/transaction.go index 0edb041..d028c7e 100644 --- a/spvwallet/client/transaction/transaction.go +++ b/spvwallet/client/transaction/transaction.go @@ -10,8 +10,7 @@ import ( "strconv" "strings" - "github.com/elastos/Elastos.ELA.SPV/log" - . "github.com/elastos/Elastos.ELA.SPV/spvwallet/cli" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/client" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" @@ -19,7 +18,7 @@ import ( "github.com/urfave/cli" ) -func CreateTransaction(c *cli.Context, wallet Wallet) error { +func CreateTransaction(c *cli.Context, wallet client.Wallet) error { txn, err := createTransaction(c, wallet) if err != nil { return err @@ -27,7 +26,7 @@ func CreateTransaction(c *cli.Context, wallet Wallet) error { return output(txn) } -func createTransaction(c *cli.Context, wallet Wallet) (*core.Transaction, error) { +func createTransaction(c *cli.Context, wallet client.Wallet) (*core.Transaction, error) { feeStr := c.String("fee") if feeStr == "" { return nil, errors.New("use --fee to specify transfer fee") @@ -40,7 +39,7 @@ func createTransaction(c *cli.Context, wallet Wallet) (*core.Transaction, error) from := c.String("from") if from == "" { - from, err = SelectAccount(wallet) + from, err = client.SelectAccount(wallet) if err != nil { return nil, err } @@ -92,7 +91,7 @@ func createTransaction(c *cli.Context, wallet Wallet) (*core.Transaction, error) return txn, nil } -func createMultiOutputTransaction(c *cli.Context, wallet Wallet, path, from string, fee *common.Fixed64) (*core.Transaction, error) { +func createMultiOutputTransaction(c *cli.Context, wallet client.Wallet, path, from string, fee *common.Fixed64) (*core.Transaction, error) { if _, err := os.Stat(path); err != nil { return nil, errors.New("invalid multi output file path") } @@ -102,7 +101,7 @@ func createMultiOutputTransaction(c *cli.Context, wallet Wallet, path, from stri } scanner := bufio.NewScanner(file) - var multiOutput []*Transfer + var multiOutput []*client.Transfer for scanner.Scan() { columns := strings.Split(scanner.Text(), ",") if len(columns) < 2 { @@ -114,8 +113,7 @@ func createMultiOutputTransaction(c *cli.Context, wallet Wallet, path, from stri return nil, errors.New("invalid multi output transaction amount: " + amountStr) } address := strings.TrimSpace(columns[0]) - multiOutput = append(multiOutput, &Transfer{address, amount}) - log.Trace("Multi output address:", address, ", amount:", amountStr) + multiOutput = append(multiOutput, &client.Transfer{address, amount}) } lockStr := c.String("lock") @@ -139,7 +137,7 @@ func createMultiOutputTransaction(c *cli.Context, wallet Wallet, path, from stri return txn, nil } -func SignTransaction(password []byte, context *cli.Context, wallet Wallet) error { +func SignTransaction(password []byte, context *cli.Context, wallet client.Wallet) error { txn, err := getTransaction(context) if err != nil { return err @@ -153,13 +151,13 @@ func SignTransaction(password []byte, context *cli.Context, wallet Wallet) error return output(txn) } -func signTransaction(password []byte, wallet Wallet, txn *core.Transaction) (*core.Transaction, error) { +func signTransaction(password []byte, wallet client.Wallet, txn *core.Transaction) (*core.Transaction, error) { haveSign, needSign, err := crypto.GetSignStatus(txn.Programs[0].Code, txn.Programs[0].Parameter) if haveSign == needSign { return nil, errors.New("transaction was fully signed, no need more sign") } - password, err = GetPassword(password, false) + password, err = client.GetPassword(password, false) if err != nil { return nil, err } @@ -167,7 +165,7 @@ func signTransaction(password []byte, wallet Wallet, txn *core.Transaction) (*co return wallet.Sign(password, txn) } -func SendTransaction(password []byte, context *cli.Context, wallet Wallet) error { +func SendTransaction(password []byte, context *cli.Context, wallet client.Wallet) error { content, err := getContent(context) var txn *core.Transaction @@ -300,7 +298,7 @@ func transactionAction(context *cli.Context) { } pass := context.String("password") - wallet, err := Open() + wallet, err := client.Open() if err != nil { fmt.Println("error: open wallet failed,", err) os.Exit(2) @@ -339,7 +337,7 @@ func NewCommand() cli.Command { Usage: "use [--create, --sign, --send], to create, sign or send a transaction", Description: "create, sign or send transaction", ArgsUsage: "[args]", - Flags: append(CommonFlags, + Flags: append(client.CommonFlags, cli.BoolFlag{ Name: "create", Usage: "use [--from] --to --amount --fee [--lock], or [--from] --file --fee [--lock]\n" + diff --git a/spvwallet/client/wallet/wallet.go b/spvwallet/client/wallet/wallet.go index 4576076..d553824 100644 --- a/spvwallet/client/wallet/wallet.go +++ b/spvwallet/client/wallet/wallet.go @@ -3,8 +3,7 @@ package wallet import ( "fmt" - . "github.com/elastos/Elastos.ELA.SPV/spvwallet" - . "github.com/elastos/Elastos.ELA.SPV/spvwallet/cli" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/client" "github.com/urfave/cli" ) @@ -13,32 +12,32 @@ func createWallet(context *cli.Context) { password := []byte(context.String("password")) var err error - password, err = GetPassword(password, true) + password, err = client.GetPassword(password, true) if err != nil { fmt.Println("--GET PASSWORD FAILED--") return } - err = Create(password) + err = client.Create(password) if err != nil { - fmt.Println("--CREAT WALLET FAILED--") + fmt.Println("--CREATE WALLET FAILED--") return } - ShowAccountInfo(password) + client.ShowAccountInfo(password) } func changePassword(context *cli.Context) { password := []byte(context.String("password")) // Verify old password - oldPassword, err := GetPassword(password, false) + oldPassword, err := client.GetPassword(password, false) if err != nil { fmt.Println("--GET PASSWORD FAILED--") return } - wallet, err := Open() + wallet, err := client.Open() if err != nil { fmt.Println("--OPEN WALLET FAILED--") return @@ -52,14 +51,14 @@ func changePassword(context *cli.Context) { // Input new password fmt.Println("--PLEASE INPUT NEW PASSWORD--") - newPassword, err := GetPassword(nil, true) + newPassword, err := client.GetPassword(nil, true) if err != nil { - fmt.Println("--GET NEW PASSWROD FAILED--") + fmt.Println("--GET NEW PASSWORD FAILED--") return } if err := wallet.ChangePassword(oldPassword, newPassword); err != nil { - fmt.Println("--CHANGED WALLET PASSWROD FAILED--") + fmt.Println("--CHANGED WALLET PASSWORD FAILED--") return } @@ -70,13 +69,13 @@ func resetDatabase(context *cli.Context) { password := []byte(context.String("password")) // Verify old password - oldPassword, err := GetPassword(password, false) + oldPassword, err := client.GetPassword(password, false) if err != nil { fmt.Println("--GET PASSWORD FAILED--") return } - wallet, err := Open() + wallet, err := client.Open() if err != nil { fmt.Println("--OPEN WALLET FAILED--") return @@ -101,7 +100,7 @@ func NewCreateCommand() cli.Command { return cli.Command{ Name: "create", Usage: "create wallet", - Flags: append(CommonFlags), + Flags: append(client.CommonFlags), Action: createWallet, OnUsageError: func(c *cli.Context, err error, subCommand bool) error { return cli.NewExitError(err, 1) @@ -113,7 +112,7 @@ func NewChangePasswordCommand() cli.Command { return cli.Command{ Name: "changepassword", Usage: "change wallet password", - Flags: append(CommonFlags), + Flags: append(client.CommonFlags), Action: changePassword, OnUsageError: func(c *cli.Context, err error, subCommand bool) error { return cli.NewExitError(err, 1) @@ -125,7 +124,7 @@ func NewResetCommand() cli.Command { return cli.Command{ Name: "reset", Usage: "reset wallet database including transactions, utxos and stxos", - Flags: append(CommonFlags), + Flags: append(client.CommonFlags), Action: resetDatabase, OnUsageError: func(c *cli.Context, err error, subCommand bool) error { return cli.NewExitError(err, 1) From 3372fb15f07037a762a5e7d13479bf00a11de4a2 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 11 Sep 2018 10:42:00 +0800 Subject: [PATCH 19/73] add DefaultPort parameter into spvwallet/config --- spvwallet/config/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spvwallet/config/config.go b/spvwallet/config/config.go index ceaf083..2705da0 100644 --- a/spvwallet/config/config.go +++ b/spvwallet/config/config.go @@ -15,12 +15,13 @@ var config *Config // The single instance of config type Config struct { Magic uint32 + SeedList []string + DefaultPort uint16 // default port for public peers to provide services. PrintLevel int MaxLogsSize int64 MaxPerLogSize int64 RPCPort uint16 Foundation string - SeedList []string } func (config *Config) readConfigFile() error { From a2ecbc6fb3fa1cf39a2c566d8a95239e4ebfb5fd Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 11 Sep 2018 10:42:59 +0800 Subject: [PATCH 20/73] adjust SendTransacction method return parameters --- spvwallet/rpc/fucntions.go | 4 ++-- spvwallet/rpc/log.go | 28 ++++++++++++++++++++++++++++ spvwallet/rpc/server.go | 5 +---- 3 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 spvwallet/rpc/log.go diff --git a/spvwallet/rpc/fucntions.go b/spvwallet/rpc/fucntions.go index 294ea46..99068f1 100644 --- a/spvwallet/rpc/fucntions.go +++ b/spvwallet/rpc/fucntions.go @@ -34,9 +34,9 @@ func (server *Server) sendTransaction(req Req) Resp { if err != nil { return FunctionError("Deserialize transaction failed") } - txId, err := server.SendTransaction(tx) + err = server.SendTransaction(tx) if err != nil { return FunctionError(err.Error()) } - return Success(txId.String()) + return Success(tx.Hash().String()) } diff --git a/spvwallet/rpc/log.go b/spvwallet/rpc/log.go new file mode 100644 index 0000000..b7412c2 --- /dev/null +++ b/spvwallet/rpc/log.go @@ -0,0 +1,28 @@ +package rpc + +import ( + "github.com/elastos/Elastos.ELA.Utility/elalog" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log elalog.Logger + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = elalog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using elalog. +func UseLogger(logger elalog.Logger) { + log = logger +} diff --git a/spvwallet/rpc/server.go b/spvwallet/rpc/server.go index 262bb38..dcdb905 100644 --- a/spvwallet/rpc/server.go +++ b/spvwallet/rpc/server.go @@ -7,9 +7,6 @@ import ( "net/http" "os" - "github.com/elastos/Elastos.ELA.SPV/log" - - "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" ) @@ -28,7 +25,7 @@ type Server struct { http.Server methods map[string]func(Req) Resp NotifyNewAddress func([]byte) - SendTransaction func(core.Transaction) (*common.Uint256, error) + SendTransaction func(core.Transaction) error } func (server *Server) Start() { From 9cacda95c96a700e710f4e86951ba05db1c69429 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 11 Sep 2018 10:44:43 +0800 Subject: [PATCH 21/73] initial refactor commit for spvwallet/store/ package --- spvwallet/store/headers/cache.go | 53 +++++ spvwallet/store/headers/database.go | 38 ++- spvwallet/store/headers/log.go | 28 +++ spvwallet/store/log.go | 16 ++ spvwallet/store/sqlite/addrs.go | 70 +++--- spvwallet/store/sqlite/addrsbatch.go | 43 ++++ spvwallet/store/sqlite/database.go | 331 ++++----------------------- spvwallet/store/sqlite/databatch.go | 80 +++++++ spvwallet/store/sqlite/interface.go | 74 +++--- spvwallet/store/sqlite/log.go | 28 +++ spvwallet/store/sqlite/state.go | 27 ++- spvwallet/store/sqlite/stxos.go | 143 ++++++------ spvwallet/store/sqlite/stxosbatch.go | 47 ++++ spvwallet/store/sqlite/txs.go | 54 +++-- spvwallet/store/sqlite/txsbatch.go | 44 ++++ spvwallet/store/sqlite/utxos.go | 113 +++++---- spvwallet/store/sqlite/utxosbatch.go | 41 ++++ 17 files changed, 715 insertions(+), 515 deletions(-) create mode 100644 spvwallet/store/headers/cache.go create mode 100644 spvwallet/store/headers/log.go create mode 100644 spvwallet/store/log.go create mode 100644 spvwallet/store/sqlite/addrsbatch.go create mode 100644 spvwallet/store/sqlite/databatch.go create mode 100644 spvwallet/store/sqlite/log.go create mode 100644 spvwallet/store/sqlite/stxosbatch.go create mode 100644 spvwallet/store/sqlite/txsbatch.go create mode 100644 spvwallet/store/sqlite/utxosbatch.go diff --git a/spvwallet/store/headers/cache.go b/spvwallet/store/headers/cache.go new file mode 100644 index 0000000..6cafdb4 --- /dev/null +++ b/spvwallet/store/headers/cache.go @@ -0,0 +1,53 @@ +package headers + +import ( + "errors" + "sync" + + "github.com/cevaris/ordered_map" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" +) + +type cache struct { + sync.RWMutex + size int + tip *util.Header + headers *ordered_map.OrderedMap +} + +func newHeaderCache(size int) *cache { + return &cache{ + size: size, + headers: ordered_map.NewOrderedMap(), + } +} + +func (cache *cache) pop() { + iter := cache.headers.IterFunc() + k, ok := iter() + if ok { + cache.headers.Delete(k.Key) + } +} + +func (cache *cache) set(header *util.Header) { + cache.Lock() + defer cache.Unlock() + + if cache.headers.Len() > cache.size { + cache.pop() + } + cache.headers.Set(header.Hash().String(), header) +} + +func (cache *cache) get(hash *common.Uint256) (*util.Header, error) { + cache.RLock() + defer cache.RUnlock() + + sh, ok := cache.headers.Get(hash.String()) + if !ok { + return nil, errors.New("Header not found in cache ") + } + return sh.(*util.Header), nil +} diff --git a/spvwallet/store/headers/database.go b/spvwallet/store/headers/database.go index c320034..803abd9 100644 --- a/spvwallet/store/headers/database.go +++ b/spvwallet/store/headers/database.go @@ -2,24 +2,22 @@ package headers import ( "encoding/hex" - "errors" "fmt" "math/big" "sync" "github.com/elastos/Elastos.ELA.SPV/database" - "github.com/elastos/Elastos.ELA.SPV/log" "github.com/elastos/Elastos.ELA.SPV/util" "github.com/boltdb/bolt" "github.com/elastos/Elastos.ELA.Utility/common" ) -// Ensure HeadersDB implement headers interface -var _ database.Headers = (*HeadersDB)(nil) +// Ensure Database implement headers interface +var _ database.Headers = (*Database)(nil) // Headers implements Headers using bolt DB -type HeadersDB struct { +type Database struct { *sync.RWMutex *bolt.DB cache *cache @@ -31,7 +29,7 @@ var ( KEYChainTip = []byte("ChainTip") ) -func NewHeadersDB() (*HeadersDB, error) { +func New() (*Database, error) { db, err := bolt.Open("headers.bin", 0644, &bolt.Options{InitialMmapSize: 5000000}) if err != nil { return nil, err @@ -49,7 +47,7 @@ func NewHeadersDB() (*HeadersDB, error) { return nil }) - headers := &HeadersDB{ + headers := &Database{ RWMutex: new(sync.RWMutex), DB: db, cache: newHeaderCache(100), @@ -60,7 +58,7 @@ func NewHeadersDB() (*HeadersDB, error) { return headers, nil } -func (h *HeadersDB) initCache() { +func (h *Database) initCache() { best, err := h.GetBest() if err != nil { return @@ -79,7 +77,7 @@ func (h *HeadersDB) initCache() { } } -func (h *HeadersDB) Put(header *util.Header, newTip bool) error { +func (h *Database) Put(header *util.Header, newTip bool) error { h.Lock() defer h.Unlock() @@ -110,14 +108,14 @@ func (h *HeadersDB) Put(header *util.Header, newTip bool) error { }) } -func (h *HeadersDB) GetPrevious(header *util.Header) (*util.Header, error) { +func (h *Database) GetPrevious(header *util.Header) (*util.Header, error) { if header.Height == 1 { return &util.Header{TotalWork: new(big.Int)}, nil } return h.Get(&header.Previous) } -func (h *HeadersDB) Get(hash *common.Uint256) (header *util.Header, err error) { +func (h *Database) Get(hash *common.Uint256) (header *util.Header, err error) { h.RLock() defer h.RUnlock() @@ -143,7 +141,7 @@ func (h *HeadersDB) Get(hash *common.Uint256) (header *util.Header, err error) { return header, err } -func (h *HeadersDB) GetBest() (header *util.Header, err error) { +func (h *Database) GetBest() (header *util.Header, err error) { h.RLock() defer h.RUnlock() @@ -152,15 +150,9 @@ func (h *HeadersDB) GetBest() (header *util.Header, err error) { } err = h.View(func(tx *bolt.Tx) error { - header, err = getHeader(tx, BKTChainTip, KEYChainTip) - if err != nil { - return err - } - - return nil + return err }) - if err != nil { return nil, fmt.Errorf("Headers db get tip error %s", err.Error()) } @@ -168,7 +160,7 @@ func (h *HeadersDB) GetBest() (header *util.Header, err error) { return header, err } -func (h *HeadersDB) Clear() error { +func (h *Database) Clear() error { h.Lock() defer h.Unlock() @@ -183,17 +175,17 @@ func (h *HeadersDB) Clear() error { } // Close db -func (h *HeadersDB) Close() error { +func (h *Database) Close() error { h.Lock() err := h.DB.Close() - log.Debug("Headers DB closed") + log.Debug("headers database closed") return err } func getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*util.Header, error) { headerBytes := tx.Bucket(bucket).Get(key) if headerBytes == nil { - return nil, errors.New(fmt.Sprintf("Header %s does not exist in database", hex.EncodeToString(key))) + return nil, fmt.Errorf("Header %s does not exist in database", hex.EncodeToString(key)) } var header util.Header diff --git a/spvwallet/store/headers/log.go b/spvwallet/store/headers/log.go new file mode 100644 index 0000000..a85039b --- /dev/null +++ b/spvwallet/store/headers/log.go @@ -0,0 +1,28 @@ +package headers + +import ( + "github.com/elastos/Elastos.ELA.Utility/elalog" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log elalog.Logger + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = elalog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using elalog. +func UseLogger(logger elalog.Logger) { + log = logger +} diff --git a/spvwallet/store/log.go b/spvwallet/store/log.go new file mode 100644 index 0000000..68c9456 --- /dev/null +++ b/spvwallet/store/log.go @@ -0,0 +1,16 @@ +package store + +import ( + "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/headers" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/sqlite" + + "github.com/elastos/Elastos.ELA.Utility/elalog" +) + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using elalog. +func UseLogger(logger elalog.Logger) { + headers.UseLogger(logger) + sqlite.UseLogger(logger) +} diff --git a/spvwallet/store/sqlite/addrs.go b/spvwallet/store/sqlite/addrs.go index 8a0fda7..aa53289 100644 --- a/spvwallet/store/sqlite/addrs.go +++ b/spvwallet/store/sqlite/addrs.go @@ -4,50 +4,49 @@ import ( "database/sql" "sync" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" ) +// Ensure addrs implement Addrs interface. +var _ Addrs = (*addrs)(nil) + const CreateAddrsDB = `CREATE TABLE IF NOT EXISTS Addrs( Hash BLOB NOT NULL PRIMARY KEY, Script BLOB, Type INTEGER NOT NULL );` -type Addrs struct { +type addrs struct { *sync.RWMutex *sql.DB } -func NewAddrs(db *sql.DB, lock *sync.RWMutex) (*Addrs, error) { +func NewAddrs(db *sql.DB, lock *sync.RWMutex) (*addrs, error) { _, err := db.Exec(CreateAddrsDB) if err != nil { return nil, err } - return &Addrs{RWMutex: lock, DB: db}, nil + return &addrs{RWMutex: lock, DB: db}, nil } // put a script to database -func (db *Addrs) Put(hash *common.Uint168, script []byte, addrType int) error { - db.Lock() - defer db.Unlock() +func (a *addrs) Put(hash *common.Uint168, script []byte, addrType int) error { + a.Lock() + defer a.Unlock() sql := "INSERT OR REPLACE INTO Addrs(Hash, Script, Type) VALUES(?,?,?)" - _, err := db.Exec(sql, hash.Bytes(), script, addrType) - if err != nil { - return err - } - - return nil + _, err := a.Exec(sql, hash.Bytes(), script, addrType) + return err } // get a script from database -func (db *Addrs) Get(hash *common.Uint168) (*util.Addr, error) { - db.RLock() - defer db.RUnlock() +func (a *addrs) Get(hash *common.Uint168) (*sutil.Addr, error) { + a.RLock() + defer a.RUnlock() - row := db.QueryRow(`SELECT Script, Type FROM Addrs WHERE Hash=?`, hash.Bytes()) + row := a.QueryRow(`SELECT Script, Type FROM Addrs WHERE Hash=?`, hash.Bytes()) var script []byte var addrType int err := row.Scan(&script, &addrType) @@ -55,16 +54,16 @@ func (db *Addrs) Get(hash *common.Uint168) (*util.Addr, error) { return nil, err } - return util.NewAddr(hash, script, addrType), nil + return sutil.NewAddr(hash, script, addrType), nil } // get all Addrs from database -func (db *Addrs) GetAll() ([]*util.Addr, error) { - db.RLock() - defer db.RUnlock() +func (a *addrs) GetAll() ([]*sutil.Addr, error) { + a.RLock() + defer a.RUnlock() - var addrs []*util.Addr - rows, err := db.Query("SELECT Hash, Script, Type FROM Addrs") + var addrs []*sutil.Addr + rows, err := a.Query("SELECT Hash, Script, Type FROM Addrs") if err != nil { return addrs, err } @@ -82,21 +81,32 @@ func (db *Addrs) GetAll() ([]*util.Addr, error) { if err != nil { return addrs, err } - addrs = append(addrs, util.NewAddr(hash, script, addrType)) + addrs = append(addrs, sutil.NewAddr(hash, script, addrType)) } return addrs, nil } // delete a script from database -func (db *Addrs) Delete(hash *common.Uint168) error { - db.Lock() - defer db.Unlock() +func (a *addrs) Del(hash *common.Uint168) error { + a.Lock() + defer a.Unlock() - _, err := db.Exec("DELETE FROM Addrs WHERE Hash=?", hash.Bytes()) + _, err := a.Exec("DELETE FROM Addrs WHERE Hash=?", hash.Bytes()) + return err +} + +func (a *addrs) Batch() AddrsBatch { + a.Lock() + defer a.Unlock() + + tx, err := a.DB.Begin() if err != nil { - return err + panic(err) } - return nil + return &addrsBatch{ + RWMutex: a.RWMutex, + Tx: tx, + } } diff --git a/spvwallet/store/sqlite/addrsbatch.go b/spvwallet/store/sqlite/addrsbatch.go new file mode 100644 index 0000000..9ad9ce5 --- /dev/null +++ b/spvwallet/store/sqlite/addrsbatch.go @@ -0,0 +1,43 @@ +package sqlite + +import ( + "database/sql" + "sync" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +// Ensure addrsBatch implement AddrsBatch interface. +var _ AddrsBatch = (*addrsBatch)(nil) + +type addrsBatch struct { + *sync.RWMutex + *sql.Tx +} + +// put a script to database +func (b *addrsBatch) Put(hash *common.Uint168, script []byte, addrType int) error { + b.Lock() + defer b.Unlock() + + sql := "INSERT OR REPLACE INTO Addrs(Hash, Script, Type) VALUES(?,?,?)" + _, err := b.Exec(sql, hash.Bytes(), script, addrType) + if err != nil { + return err + } + + return nil +} + +// delete a script from database +func (b *addrsBatch) Del(hash *common.Uint168) error { + b.Lock() + defer b.Unlock() + + _, err := b.Exec("DELETE FROM Addrs WHERE Hash=?", hash.Bytes()) + if err != nil { + return err + } + + return nil +} diff --git a/spvwallet/store/sqlite/database.go b/spvwallet/store/sqlite/database.go index ebe06d7..b7bc67a 100644 --- a/spvwallet/store/sqlite/database.go +++ b/spvwallet/store/sqlite/database.go @@ -1,15 +1,10 @@ -package db +package sqlite import ( "database/sql" "fmt" - "github.com/elastos/Elastos.ELA.SPV/util" "sync" - "github.com/elastos/Elastos.ELA.SPV/database" - "github.com/elastos/Elastos.ELA.SPV/log" - - "github.com/elastos/Elastos.ELA.Utility/common" _ "github.com/mattn/go-sqlite3" ) @@ -18,21 +13,21 @@ const ( DBName = "./spv_wallet.db" ) -// Ensure SQLiteDB implement TxsDB interface. -var _ database.TxsDB = (*SQLiteDB)(nil) +// Ensure database implement DataStore interface +var _ DataStore = (*database)(nil) -type SQLiteDB struct { +type database struct { *sync.RWMutex *sql.DB - state *StateDB - addrs *AddrsDB - txs *TxsDB - utxos *UTXOsDB - stxos *STXOsDB + state *state + addrs *addrs + txs *txs + utxos *utxos + stxos *stxos } -func NewSQLiteDB() (*SQLiteDB, error) { +func New() (*database, error) { db, err := sql.Open(DriverName, DBName) if err != nil { fmt.Println("Open sqlite db error:", err) @@ -42,319 +37,86 @@ func NewSQLiteDB() (*SQLiteDB, error) { lock := new(sync.RWMutex) // Create state db - stateDB, err := NewStateDB(db, lock) + state, err := NewState(db, lock) if err != nil { return nil, err } // Create addrs db - addrsDB, err := NewAddrsDB(db, lock) + addrs, err := NewAddrs(db, lock) if err != nil { return nil, err } // Create UTXOs db - utxosDB, err := NewUTXOsDB(db, lock) + utxos, err := NewUTXOs(db, lock) if err != nil { return nil, err } // Create STXOs db - stxosDB, err := NewSTXOsDB(db, lock) + stxos, err := NewSTXOs(db, lock) if err != nil { return nil, err } // Create Txs db - txnsDB, err := NewTxsDB(db, lock) + txns, err := NewTxs(db, lock) if err != nil { return nil, err } - return &SQLiteDB{ + return &database{ RWMutex: lock, DB: db, - state: stateDB, - addrs: addrsDB, - utxos: utxosDB, - stxos: stxosDB, - txs: txnsDB, + state: state, + addrs: addrs, + utxos: utxos, + stxos: stxos, + txs: txns, }, nil } -func (db *SQLiteDB) Chain() Chain { - return db.state -} - -func (db *SQLiteDB) Addrs() Addrs { - return db.addrs -} - -func (db *SQLiteDB) Txs() Txs { - return db.txs -} - -func (db *SQLiteDB) UTXOs() UTXOs { - return db.utxos -} - -func (db *SQLiteDB) STXOs() STXOs { - return db.stxos -} - -// checkDoubleSpends takes a transaction and compares it with -// all transactions in the db. It returns a slice of all txIds in the db -// which are double spent by the received tx. -func (wallet *SQLiteDB) checkDoubleSpends(tx *util.Tx) ([]*common.Uint256, error) { - var dubs []*common.Uint256 - txId := tx.Hash() - txs, err := wallet.dataStore.Txs().GetAll() - if err != nil { - return nil, err - } - for _, compTx := range txs { - // Skip coinbase transaction - if compTx.Data.IsCoinBaseTx() { - continue - } - // Skip duplicate transaction - compTxId := compTx.Data.Hash() - if compTxId.IsEqual(txId) { - continue - } - for _, txIn := range tx.Inputs { - for _, compIn := range compTx.Data.Inputs { - if txIn.Previous.IsEqual(compIn.Previous) { - // Found double spend - dubs = append(dubs, &compTxId) - break // back to txIn loop - } - } - } - } - return dubs, nil -} - -// Batch returns a TxBatch instance for transactions batch -// commit, this can get better performance when commit a bunch -// of transactions within a block. -func (db *SQLiteDB) Batch() database.TxBatch { - -} - -// CommitTx save a transaction into database, and return -// if it is a false positive and error. -func (db *SQLiteDB) CommitTx(tx *util.Tx) (bool, error) { - txId := tx.Hash() - height := tx.Height - - sh, ok := db.txIds.Get(txId) - if ok && (sh > 0 || (sh == 0 && height == 0)) { - return false, nil - } - - dubs, err := db.checkDoubleSpends(tx) - if err != nil { - return false, nil - } - if len(dubs) > 0 { - if height == 0 { - return false, nil - } else { - // Rollback any double spend transactions - for _, dub := range dubs { - if err := db.RollbackTx(dub); err != nil { - return false, nil - } - } - } - } - - hits := 0 - // Save UTXOs - for index, output := range tx.Outputs { - // Filter address - if wallet.getAddrFilter().ContainAddr(output.ProgramHash) { - var lockTime uint32 - if tx.TxType == core.CoinBase { - lockTime = height + 100 - } - utxo := ToUTXO(txId, height, index, output.Value, lockTime) - err := wallet.dataStore.UTXOs().Put(&output.ProgramHash, utxo) - if err != nil { - return false, err - } - hits++ - } - } - - // Put spent UTXOs to STXOs - for _, input := range tx.Inputs { - // Try to move UTXO to STXO, if a UTXO in database was spent, it will be moved to STXO - err := wallet.dataStore.STXOs().FromUTXO(&input.Previous, &txId, height) - if err == nil { - hits++ - } - } - - // If no hits, no need to save transaction - if hits == 0 { - return true, nil - } - - // Save transaction - err := wallet.dataStore.Txs().Put(db.NewTx(*tx, height)) - if err != nil { - return false, err - } - - wallet.txIds.Add(txId, height) - - return false, nil -} - -// HaveTx returns if the transaction already saved in database -// by it's id. -func (db *SQLiteDB) HaveTx(txId *common.Uint256) (bool, error) { - +func (d *database) State() State { + return d.state } -// GetTxs returns all transactions within the given height. -func (db *SQLiteDB) GetTxs(height uint32) ([]*util.Tx, error) { - +func (d *database) Addrs() Addrs { + return d.addrs } -// RemoveTxs delete all transactions on the given height. Return -// how many transactions are deleted from database. -func (db *SQLiteDB) RemoveTxs(height uint32) (int, error) { - +func (d *database) Txs() Txs { + return d.txs } -func (db *SQLiteDB) Rollback(height uint32) error { - db.Lock() - defer db.Unlock() - - tx, err := db.Begin() - if err != nil { - return err - } - defer tx.Rollback() - - // Rollback UTXOs - _, err = tx.Exec("DELETE FROM UTXOs WHERE AtHeight=?", height) - if err != nil { - return err - } - - // Rollback STXOs, move UTXOs back first, then delete the STXOs - _, err = tx.Exec(`INSERT OR REPLACE INTO UTXOs(OutPoint, Value, LockTime, AtHeight, ScriptHash) - SELECT OutPoint, Value, LockTime, AtHeight, ScriptHash FROM STXOs WHERE SpendHeight=?`, height) - if err != nil { - return err - } - _, err = tx.Exec("DELETE FROM STXOs WHERE SpendHeight=?", height) - if err != nil { - return err - } - - // Rollback TXNs - _, err = tx.Exec("DELETE FROM TXNs WHERE Height=?", height) - if err != nil { - return err - } - - return tx.Commit() +func (d *database) UTXOs() UTXOs { + return d.utxos } -func (db *SQLiteDB) RollbackTx(txId *common.Uint256) error { - db.Lock() - defer db.Unlock() - - return db.rollbackTx(txId) +func (d *database) STXOs() STXOs { + return d.stxos } -func (db *SQLiteDB) rollbackTx(txId *common.Uint256) error { - tx, err := db.Begin() - if err != nil { - return err - } - defer tx.Rollback() - - // Get unconfirmed STXOs - rows, err := db.Query( - "SELECT OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight FROM STXOs WHERE SpendHeight=?", 0) - if err != nil { - return err - } - defer rows.Close() +func (d *database) Batch() DataBatch { + d.Lock() + defer d.Unlock() - stxos, err := db.stxos.getSTXOs(rows) + tx, err := d.Begin() if err != nil { - return err + panic(err) } - for _, stxo := range stxos { - outpoint := stxo.Op.Bytes() - if txId.IsEqual(stxo.SpendTxId) { - // Restore UTXO - _, err = tx.Exec(`INSERT OR REPLACE INTO UTXOs(OutPoint, Value, LockTime, AtHeight, ScriptHash) - SELECT OutPoint, Value, LockTime, AtHeight, ScriptHash FROM STXOs WHERE OutPoint=?`, outpoint) - if err != nil { - return err - } - // Delele STXO - _, err = tx.Exec("DELETE FROM STXOs WHERE OutPoint=?", outpoint) - if err != nil { - return err - } - } - if txId.IsEqual(stxo.UTXO.Op.TxID) { - // Delele STXO - _, err = tx.Exec("DELETE FROM STXOs WHERE OutPoint=?", outpoint) - if err != nil { - return err - } - if err := db.rollbackTx(&stxo.SpendTxId); err != nil { - return err - } - } + return &dataBatch{ + RWMutex: d.RWMutex, + Tx: tx, } - // Get unconfirmed UTXOs - rows, err = db.Query("SELECT OutPoint, Value, LockTime, AtHeight FROM UTXOs WHERE AtHeight=?", 0) - if err != nil { - return err - } - defer rows.Close() - - utxos, err := db.utxos.getUTXOs(rows) - if err != nil { - return err - } - - for _, utxo := range utxos { - if txId.IsEqual(utxo.Op.TxID) { - // Delele UTXO - _, err = tx.Exec("DELETE FROM UTXOs WHERE OutPoint=?", utxo.Op.Bytes()) - if err != nil { - return err - } - } - } - - // Delele transaction - _, err = tx.Exec("DELETE FROM TXNs WHERE Hash=?", txId.Bytes()) - if err != nil { - return err - } - - return tx.Commit() } -func (db *SQLiteDB) Clear() error { - tx, err := db.Begin() +func (d *database) Clear() error { + tx, err := d.Begin() if err != nil { return err } // Drop all tables except Addrs - _, err = tx.Exec(`DROP TABLE IF EXISTS Chain; + _, err = tx.Exec(`DROP TABLE IF EXISTS State; DROP TABLE IF EXISTS UTXOs; DROP TABLE IF EXISTS STXOs; DROP TABLE IF EXISTS TXNs;`) @@ -365,8 +127,9 @@ func (db *SQLiteDB) Clear() error { return tx.Commit() } -func (db *SQLiteDB) Close() { - db.Lock() - db.DB.Close() - log.Debug("SQLite DB closed") +func (d *database) Close() error { + d.Lock() + err := d.DB.Close() + log.Debug("sqlite database closed") + return err } diff --git a/spvwallet/store/sqlite/databatch.go b/spvwallet/store/sqlite/databatch.go new file mode 100644 index 0000000..6c44ec0 --- /dev/null +++ b/spvwallet/store/sqlite/databatch.go @@ -0,0 +1,80 @@ +package sqlite + +import ( + "database/sql" + "sync" +) + +// Ensure dataBatch implement DataStore interface +var _ DataBatch = (*dataBatch)(nil) + +type dataBatch struct { + *sync.RWMutex + *sql.Tx +} + +func (d *dataBatch) Addrs() AddrsBatch { + d.Lock() + defer d.Unlock() + + return &addrsBatch{ + RWMutex: d.RWMutex, + Tx: d.Tx, + } +} + +func (d *dataBatch) Txs() TxsBatch { + d.Lock() + defer d.Unlock() + + return &txsBatch{ + RWMutex: d.RWMutex, + Tx: d.Tx, + } +} + +func (d *dataBatch) UTXOs() UTXOsBatch { + d.Lock() + defer d.Unlock() + + return &utxosBatch{ + RWMutex: d.RWMutex, + Tx: d.Tx, + } +} + +func (d *dataBatch) STXOs() STXOsBatch { + d.Lock() + defer d.Unlock() + + return &stxosBatch{ + RWMutex: d.RWMutex, + Tx: d.Tx, + } +} + +func (d *dataBatch) RollbackHeight(height uint32) error { + d.Lock() + defer d.Unlock() + + // Rollback UTXOs + _, err := d.Exec("DELETE FROM UTXOs WHERE AtHeight=?", height) + if err != nil { + return err + } + + // Rollback STXOs, move UTXOs back first, then delete the STXOs + _, err = d.Exec(`INSERT OR REPLACE INTO UTXOs(OutPoint, Value, LockTime, AtHeight, ScriptHash) + SELECT OutPoint, Value, LockTime, AtHeight, ScriptHash FROM STXOs WHERE SpendHeight=?`, height) + if err != nil { + return err + } + _, err = d.Exec("DELETE FROM STXOs WHERE SpendHeight=?", height) + if err != nil { + return err + } + + // Rollback TXNs + _, err = d.Exec("DELETE FROM TXNs WHERE Height=?", height) + return err +} \ No newline at end of file diff --git a/spvwallet/store/sqlite/interface.go b/spvwallet/store/sqlite/interface.go index 5f3fb60..7552794 100644 --- a/spvwallet/store/sqlite/interface.go +++ b/spvwallet/store/sqlite/interface.go @@ -1,7 +1,7 @@ -package store +package sqlite import ( - "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" @@ -18,16 +18,18 @@ type DataStore interface { Close() error } -type Batch interface { - Rollback() error - Commit() error -} - type DataBatch interface { + batch Addrs() AddrsBatch Txs() TxsBatch UTXOs() UTXOsBatch STXOs() STXOsBatch + RollbackHeight(height uint32) error +} + +type batch interface { + Rollback() error + Commit() error } type State interface { @@ -43,17 +45,20 @@ type Addrs interface { Put(hash *common.Uint168, script []byte, addrType int) error // get a address from database - Get(hash *common.Uint168) (*util.Addr, error) + Get(hash *common.Uint168) (*sutil.Addr, error) // get all addresss from database - GetAll() ([]*util.Addr, error) + GetAll() ([]*sutil.Addr, error) // delete a address from database Del(hash *common.Uint168) error + + // Batch return a AddrsBatch + Batch() AddrsBatch } type AddrsBatch interface { - Batch + batch // put a address to database Put(hash *common.Uint168, script []byte, addrType int) error @@ -64,53 +69,59 @@ type AddrsBatch interface { type Txs interface { // Put a new transaction to database - Put(txn *util.Tx) error + Put(txn *sutil.Tx) error // Fetch a raw tx and it's metadata given a hash - Get(txId *common.Uint256) (*util.Tx, error) + Get(txId *common.Uint256) (*sutil.Tx, error) // Fetch all transactions from database - GetAll() ([]*util.Tx, error) + GetAll() ([]*sutil.Tx, error) + + // Fetch all unconfirmed transactions. + GetAllUnconfirmed()([]*sutil.Tx, error) // Delete a transaction from the db Del(txId *common.Uint256) error + + // Batch return a TxsBatch + Batch() TxsBatch } type TxsBatch interface { - Batch + batch // Put a new transaction to database - Put(txn *util.Tx) error + Put(txn *sutil.Tx) error // Delete a transaction from the db Del(txId *common.Uint256) error - - // Delete transactions on the given height. - DelAll(height uint32) error } type UTXOs interface { // put a utxo to database - Put(hash *common.Uint168, utxo *util.UTXO) error + Put(utxo *sutil.UTXO) error // get a utxo from database - Get(op *core.OutPoint) (*util.UTXO, error) + Get(op *core.OutPoint) (*sutil.UTXO, error) // get utxos of the given address hash from database - GetAddrAll(hash *common.Uint168) ([]*util.UTXO, error) + GetAddrAll(hash *common.Uint168) ([]*sutil.UTXO, error) // Get all UTXOs in database - GetAll() ([]*util.UTXO, error) + GetAll() ([]*sutil.UTXO, error) // delete a utxo from database Del(outPoint *core.OutPoint) error + + // Batch return a UTXOsBatch. + Batch() UTXOsBatch } type UTXOsBatch interface { - Batch + batch // put a utxo to database - Put(hash *common.Uint168, utxo *util.UTXO) error + Put(utxo *sutil.UTXO) error // delete a utxo from database Del(outPoint *core.OutPoint) error @@ -118,26 +129,29 @@ type UTXOsBatch interface { type STXOs interface { // Put save a STXO into database - Put(stxo *util.STXO) error + Put(stxo *sutil.STXO) error // get a stxo from database - Get(op *core.OutPoint) (*util.STXO, error) + Get(op *core.OutPoint) (*sutil.STXO, error) // get stxos of the given address hash from database - GetAddrAll(hash *common.Uint168) ([]*util.STXO, error) + GetAddrAll(hash *common.Uint168) ([]*sutil.STXO, error) // Get all STXOs in database - GetAll() ([]*util.STXO, error) + GetAll() ([]*sutil.STXO, error) // delete a stxo from database Del(outPoint *core.OutPoint) error + + // Batch return a STXOsBatch. + Batch() STXOsBatch } type STXOsBatch interface { - Batch + batch // Put save a STXO into database - Put(stxo *util.STXO) error + Put(stxo *sutil.STXO) error // delete a stxo from database Del(outPoint *core.OutPoint) error diff --git a/spvwallet/store/sqlite/log.go b/spvwallet/store/sqlite/log.go new file mode 100644 index 0000000..93c37c0 --- /dev/null +++ b/spvwallet/store/sqlite/log.go @@ -0,0 +1,28 @@ +package sqlite + +import ( + "github.com/elastos/Elastos.ELA.Utility/elalog" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log elalog.Logger + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = elalog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using elalog. +func UseLogger(logger elalog.Logger) { + log = logger +} diff --git a/spvwallet/store/sqlite/state.go b/spvwallet/store/sqlite/state.go index dbbe454..268d11e 100644 --- a/spvwallet/store/sqlite/state.go +++ b/spvwallet/store/sqlite/state.go @@ -1,4 +1,4 @@ -package db +package sqlite import ( "database/sql" @@ -14,25 +14,28 @@ const ( HeightKey = "Height" ) -type StateDB struct { +// Ensure state implement State interface. +var _ State = (*state)(nil) + +type state struct { *sync.RWMutex *sql.DB } -func NewStateDB(db *sql.DB, lock *sync.RWMutex) (*StateDB, error) { +func NewState(db *sql.DB, lock *sync.RWMutex) (*state, error) { _, err := db.Exec(CreateStateDB) if err != nil { return nil, err } - return &StateDB{RWMutex: lock, DB: db}, nil + return &state{RWMutex: lock, DB: db}, nil } // get state height -func (db *StateDB) GetHeight() uint32 { - db.RLock() - defer db.RUnlock() +func (s *state) GetHeight() uint32 { + s.RLock() + defer s.RUnlock() - row := db.QueryRow("SELECT Value FROM State WHERE Key=?", HeightKey) + row := s.QueryRow("SELECT Value FROM State WHERE Key=?", HeightKey) var height uint32 err := row.Scan(&height) @@ -44,9 +47,9 @@ func (db *StateDB) GetHeight() uint32 { } // save state height -func (db *StateDB) PutHeight(height uint32) { - db.Lock() - defer db.Unlock() +func (s *state) PutHeight(height uint32) { + s.Lock() + defer s.Unlock() - db.Exec("INSERT OR REPLACE INTO State(Key, Value) VALUES(?,?)", HeightKey, height) + s.Exec("INSERT OR REPLACE INTO State(Key, Value) VALUES(?,?)", HeightKey, height) } diff --git a/spvwallet/store/sqlite/stxos.go b/spvwallet/store/sqlite/stxos.go index b78b704..4c84fb7 100644 --- a/spvwallet/store/sqlite/stxos.go +++ b/spvwallet/store/sqlite/stxos.go @@ -2,9 +2,8 @@ package sqlite import ( "database/sql" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" "sync" - "fmt" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" @@ -17,120 +16,109 @@ const CreateSTXOsDB = `CREATE TABLE IF NOT EXISTS STXOs( AtHeight INTEGER NOT NULL, SpendHash BLOB NOT NULL, SpendHeight INTEGER NOT NULL, - ScriptHash BLOB NOT NULL + Address BLOB NOT NULL );` -type STXOs struct { +// Ensure stxos implement STXOs interface. +var _ STXOs = (*stxos)(nil) + +type stxos struct { *sync.RWMutex *sql.DB } -func NewSTXOs(db *sql.DB, lock *sync.RWMutex) (*STXOs, error) { +func NewSTXOs(db *sql.DB, lock *sync.RWMutex) (*stxos, error) { _, err := db.Exec(CreateSTXOsDB) if err != nil { return nil, err } - return &STXOs{RWMutex: lock, DB: db}, nil + return &stxos{RWMutex: lock, DB: db}, nil } -// Move a UTXO to STXO -func (db *STXOs) FromUTXO(outPoint *core.OutPoint, spendTxId *common.Uint256, spendHeight uint32) error { - db.Lock() - defer db.Unlock() - - tx, err := db.Begin() - if err != nil { - return err - } - defer tx.Rollback() - - sql := `INSERT OR REPLACE INTO STXOs(OutPoint, Value, LockTime, AtHeight, ScriptHash, SpendHash, SpendHeight) - SELECT UTXOs.OutPoint, UTXOs.Value, UTXOs.LockTime, UTXOs.AtHeight, UTXOs.ScriptHash, ?, ? FROM UTXOs - WHERE OutPoint=?` - result, err := tx.Exec(sql, spendTxId.Bytes(), spendHeight, outPoint.Bytes()) - if err != nil { - return err - } - - rows, err := result.RowsAffected() - if err != nil { - return nil - } - - if rows == 0 { - return fmt.Errorf("no rows effected") - } +// Put save a UTXO into database +func (s *stxos) Put(stxo *sutil.STXO) error { + s.Lock() + defer s.Unlock() - _, err = tx.Exec("DELETE FROM UTXOs WHERE OutPoint=?", outPoint.Bytes()) + valueBytes, err := stxo.Value.Bytes() if err != nil { return err } - - return tx.Commit() + sql := `INSERT OR REPLACE INTO STXOs(OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight, Address) + VALUES(?,?,?,?,?,?,?)` + _, err = s.Exec(sql, stxo.Op.Bytes(), valueBytes, stxo.LockTime, stxo.AtHeight, + stxo.SpendTxId.Bytes(), stxo.SpendHeight, stxo.Address.Bytes()) + return err } // get a stxo from database -func (db *STXOs) Get(outPoint *core.OutPoint) (*util.STXO, error) { - db.RLock() - defer db.RUnlock() +func (s *stxos) Get(outPoint *core.OutPoint) (*sutil.STXO, error) { + s.RLock() + defer s.RUnlock() - sql := `SELECT Value, LockTime, AtHeight, SpendHash, SpendHeight FROM STXOs WHERE OutPoint=?` - row := db.QueryRow(sql, outPoint.Bytes()) + sql := `SELECT Value, LockTime, AtHeight, SpendHash, SpendHeight, Address FROM STXOs WHERE OutPoint=?` + row := s.QueryRow(sql, outPoint.Bytes()) var valueBytes []byte var lockTime uint32 var atHeight uint32 var spendHashBytes []byte var spendHeight uint32 - err := row.Scan(&valueBytes, &lockTime, &atHeight, &spendHashBytes, &spendHeight) + var addressBytes []byte + err := row.Scan(&valueBytes, &lockTime, &atHeight, &spendHashBytes, &spendHeight, &addressBytes) if err != nil { return nil, err } - var value *common.Fixed64 - value, err = common.Fixed64FromBytes(valueBytes) + value, err := common.Fixed64FromBytes(valueBytes) if err != nil { return nil, err } - var utxo = util.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight} + address, err := common.Uint168FromBytes(addressBytes) + if err != nil { + return nil, err + } + + var utxo = sutil.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address} spendHash, err := common.Uint256FromBytes(spendHashBytes) if err != nil { return nil, err } - return &util.STXO{UTXO: utxo, SpendTxId: *spendHash, SpendHeight: spendHeight}, nil + return &sutil.STXO{UTXO: utxo, SpendTxId: *spendHash, SpendHeight: spendHeight}, nil } // get stxos of the given script hash from database -func (db *STXOs) GetAddrAll(hash *common.Uint168) ([]*util.STXO, error) { - db.RLock() - defer db.RUnlock() +func (s *stxos) GetAddrAll(hash *common.Uint168) ([]*sutil.STXO, error) { + s.RLock() + defer s.RUnlock() - sql := "SELECT OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight FROM STXOs WHERE ScriptHash=?" - rows, err := db.Query(sql, hash.Bytes()) + sql := "SELECT OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight, Address FROM STXOs WHERE Address=?" + rows, err := s.Query(sql, hash.Bytes()) if err != nil { return nil, err } defer rows.Close() - return db.getSTXOs(rows) + return s.getSTXOs(rows) } -func (db *STXOs) GetAll() ([]*util.STXO, error) { - db.RLock() - defer db.RUnlock() +func (s *stxos) GetAll() ([]*sutil.STXO, error) { + s.RLock() + defer s.RUnlock() - rows, err := db.Query("SELECT OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight FROM STXOs") + sql := "SELECT OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight, Address FROM STXOs" + rows, err := s.Query(sql) if err != nil { return nil, err } defer rows.Close() - return db.getSTXOs(rows) + return s.getSTXOs(rows) } -func (db *STXOs) getSTXOs(rows *sql.Rows) ([]*util.STXO, error) { - var stxos []*util.STXO +func (s *stxos) getSTXOs(rows *sql.Rows) ([]*sutil.STXO, error) { + var stxos []*sutil.STXO for rows.Next() { var opBytes []byte var valueBytes []byte @@ -138,7 +126,8 @@ func (db *STXOs) getSTXOs(rows *sql.Rows) ([]*util.STXO, error) { var atHeight uint32 var spendHashBytes []byte var spendHeight uint32 - err := rows.Scan(&opBytes, &valueBytes, &lockTime, &atHeight, &spendHashBytes, &spendHeight) + var addressBytes []byte + err := rows.Scan(&opBytes, &valueBytes, &lockTime, &atHeight, &spendHashBytes, &spendHeight, &addressBytes) if err != nil { return stxos, err } @@ -147,32 +136,46 @@ func (db *STXOs) getSTXOs(rows *sql.Rows) ([]*util.STXO, error) { if err != nil { return stxos, err } - var value *common.Fixed64 - value, err = common.Fixed64FromBytes(valueBytes) + value, err := common.Fixed64FromBytes(valueBytes) if err != nil { return stxos, err } - var utxo = util.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight} + address, err := common.Uint168FromBytes(addressBytes) + if err != nil { + return stxos, err + } + var utxo = sutil.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address} spendHash, err := common.Uint256FromBytes(spendHashBytes) if err != nil { return stxos, err } - stxos = append(stxos, &util.STXO{UTXO: utxo, SpendTxId: *spendHash, SpendHeight: spendHeight}) + stxos = append(stxos, &sutil.STXO{UTXO: utxo, SpendTxId: *spendHash, SpendHeight: spendHeight}) } return stxos, nil } // delete a stxo from database -func (db *STXOs) Delete(outPoint *core.OutPoint) error { - db.Lock() - defer db.Unlock() +func (s *stxos) Del(outPoint *core.OutPoint) error { + s.Lock() + defer s.Unlock() + + _, err := s.Exec("DELETE FROM STXOs WHERE OutPoint=?", outPoint.Bytes()) + return err +} - _, err := db.Exec("DELETE FROM STXOs WHERE OutPoint=?", outPoint.Bytes()) +func (s *stxos) Batch() STXOsBatch { + s.Lock() + defer s.Unlock() + + tx, err := s.DB.Begin() if err != nil { - return err + panic(err) } - return nil + return &stxosBatch{ + RWMutex: s.RWMutex, + Tx: tx, + } } diff --git a/spvwallet/store/sqlite/stxosbatch.go b/spvwallet/store/sqlite/stxosbatch.go new file mode 100644 index 0000000..45bc18b --- /dev/null +++ b/spvwallet/store/sqlite/stxosbatch.go @@ -0,0 +1,47 @@ +package sqlite + +import ( + "database/sql" + "sync" + + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + + "github.com/elastos/Elastos.ELA/core" +) + +// Ensure stxos implement STXOs interface. +var _ STXOsBatch = (*stxosBatch)(nil) + +type stxosBatch struct { + *sync.RWMutex + *sql.Tx +} + +// Put save a UTXO into database +func (s *stxosBatch) Put(stxo *sutil.STXO) error { + s.Lock() + defer s.Unlock() + + valueBytes, err := stxo.Value.Bytes() + if err != nil { + return err + } + sql := `INSERT OR REPLACE INTO STXOs(OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight, Address) + VALUES(?,?,?,?,?,?,?)` + _, err = s.Exec(sql, stxo.Op.Bytes(), valueBytes, stxo.LockTime, stxo.AtHeight, + stxo.SpendTxId.Bytes(), stxo.SpendHeight, stxo.Address.Bytes()) + return err +} + +// delete a stxo from database +func (sb *stxosBatch) Del(outPoint *core.OutPoint) error { + sb.Lock() + defer sb.Unlock() + + _, err := sb.Exec("DELETE FROM STXOs WHERE OutPoint=?", outPoint.Bytes()) + if err != nil { + return err + } + + return nil +} diff --git a/spvwallet/store/sqlite/txs.go b/spvwallet/store/sqlite/txs.go index aea3a09..04f275e 100644 --- a/spvwallet/store/sqlite/txs.go +++ b/spvwallet/store/sqlite/txs.go @@ -7,7 +7,7 @@ import ( "sync" "time" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" @@ -20,21 +20,24 @@ const CreateTxsDB = `CREATE TABLE IF NOT EXISTS Txs( RawData BLOB NOT NULL );` -type TXs struct { +// Ensure txs implement Txs interface. +var _ Txs = (*txs)(nil) + +type txs struct { *sync.RWMutex *sql.DB } -func NewTxs(db *sql.DB, lock *sync.RWMutex) (*TXs, error) { +func NewTxs(db *sql.DB, lock *sync.RWMutex) (*txs, error) { _, err := db.Exec(CreateTxsDB) if err != nil { return nil, err } - return &TXs{RWMutex: lock, DB: db}, nil + return &txs{RWMutex: lock, DB: db}, nil } // Put a new transaction to database -func (t *TXs) Put(storeTx *util.Tx) error { +func (t *txs) Put(storeTx *sutil.Tx) error { t.Lock() defer t.Unlock() @@ -46,15 +49,11 @@ func (t *TXs) Put(storeTx *util.Tx) error { sql := `INSERT OR REPLACE INTO Txs(Hash, Height, Timestamp, RawData) VALUES(?,?,?,?)` _, err = t.Exec(sql, storeTx.TxId.Bytes(), storeTx.Height, storeTx.Timestamp.Unix(), buf.Bytes()) - if err != nil { - return err - } - - return nil + return err } // Fetch a raw tx and it's metadata given a hash -func (t *TXs) Get(txId *common.Uint256) (*util.Tx, error) { +func (t *txs) Get(txId *common.Uint256) (*sutil.Tx, error) { t.RLock() defer t.RUnlock() @@ -72,16 +71,22 @@ func (t *TXs) Get(txId *common.Uint256) (*util.Tx, error) { return nil, err } - return &util.Tx{TxId: *txId, Height: height, Timestamp: time.Unix(timestamp, 0), Data: tx}, nil + return &sutil.Tx{TxId: *txId, Height: height, Timestamp: time.Unix(timestamp, 0), Data: tx}, nil } // Fetch all transactions from database -func (t *TXs) GetAll() ([]*util.Tx, error) { +func (t *txs) GetAll() ([]*sutil.Tx, error) { return t.GetAllFrom(math.MaxUint32) } +// Fetch all unconfirmed transactions from database +func (t *txs) GetAllUnconfirmed() ([]*sutil.Tx, error) { + // TODO implement memcache to get better performance. + return t.GetAllFrom(0) +} + // Fetch all transactions from the given height -func (t *TXs) GetAllFrom(height uint32) ([]*util.Tx, error) { +func (t *txs) GetAllFrom(height uint32) ([]*sutil.Tx, error) { t.RLock() defer t.RUnlock() @@ -89,7 +94,7 @@ func (t *TXs) GetAllFrom(height uint32) ([]*util.Tx, error) { if height != math.MaxUint32 { sql += " WHERE Height=?" } - var txns []*util.Tx + var txns []*sutil.Tx rows, err := t.Query(sql, height) if err != nil { return txns, err @@ -117,21 +122,32 @@ func (t *TXs) GetAllFrom(height uint32) ([]*util.Tx, error) { return nil, err } - txns = append(txns, &util.Tx{TxId: *txId, Height: height, Timestamp: time.Unix(timestamp, 0), Data: tx}) + txns = append(txns, &sutil.Tx{TxId: *txId, Height: height, Timestamp: time.Unix(timestamp, 0), Data: tx}) } return txns, nil } // Delete a transaction from the db -func (t *TXs) Delete(txId *common.Uint256) error { +func (t *txs) Del(txId *common.Uint256) error { t.Lock() defer t.Unlock() _, err := t.Exec("DELETE FROM Txs WHERE Hash=?", txId.Bytes()) + return err +} + +func (t *txs) Batch() TxsBatch { + t.Lock() + defer t.Unlock() + + tx, err := t.DB.Begin() if err != nil { - return err + panic(err) } - return nil + return &txsBatch{ + RWMutex: t.RWMutex, + Tx: tx, + } } diff --git a/spvwallet/store/sqlite/txsbatch.go b/spvwallet/store/sqlite/txsbatch.go new file mode 100644 index 0000000..3a47510 --- /dev/null +++ b/spvwallet/store/sqlite/txsbatch.go @@ -0,0 +1,44 @@ +package sqlite + +import ( + "bytes" + "database/sql" + "sync" + + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +// Ensure txs implement Txs interface. +var _ TxsBatch = (*txsBatch)(nil) + +type txsBatch struct { + *sync.RWMutex + *sql.Tx +} + +// Put a new transaction to database +func (t *txsBatch) Put(storeTx *sutil.Tx) error { + t.Lock() + defer t.Unlock() + + buf := new(bytes.Buffer) + err := storeTx.Data.SerializeUnsigned(buf) + if err != nil { + return err + } + + sql := `INSERT OR REPLACE INTO Txs(Hash, Height, Timestamp, RawData) VALUES(?,?,?,?)` + _, err = t.Exec(sql, storeTx.TxId.Bytes(), storeTx.Height, storeTx.Timestamp.Unix(), buf.Bytes()) + return err +} + +// Delete a transaction from the db +func (t *txsBatch) Del(txId *common.Uint256) error { + t.Lock() + defer t.Unlock() + + _, err := t.Exec("DELETE FROM Txs WHERE Hash=?", txId.Bytes()) + return err +} diff --git a/spvwallet/store/sqlite/utxos.go b/spvwallet/store/sqlite/utxos.go index 539c9ce..533bdd4 100644 --- a/spvwallet/store/sqlite/utxos.go +++ b/spvwallet/store/sqlite/utxos.go @@ -4,7 +4,7 @@ import ( "database/sql" "sync" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/util" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" @@ -15,99 +15,103 @@ const CreateUTXOsDB = `CREATE TABLE IF NOT EXISTS UTXOs( Value BLOB NOT NULL, LockTime INTEGER NOT NULL, AtHeight INTEGER NOT NULL, - ScriptHash BLOB NOT NULL + Address BLOB NOT NULL );` -type UTXOs struct { +// Ensure utxos implement UTXOs interface. +var _ UTXOs = (*utxos)(nil) + +type utxos struct { *sync.RWMutex *sql.DB } -func NewUTXOs(db *sql.DB, lock *sync.RWMutex) (*UTXOs, error) { +func NewUTXOs(db *sql.DB, lock *sync.RWMutex) (*utxos, error) { _, err := db.Exec(CreateUTXOsDB) if err != nil { return nil, err } - return &UTXOs{RWMutex: lock, DB: db}, nil + return &utxos{RWMutex: lock, DB: db}, nil } // put a utxo to database -func (db *UTXOs) Put(hash *common.Uint168, utxo *util.UTXO) error { - db.Lock() - defer db.Unlock() +func (u *utxos) Put(utxo *sutil.UTXO) error { + u.Lock() + defer u.Unlock() valueBytes, err := utxo.Value.Bytes() if err != nil { return err } - sql := "INSERT OR REPLACE INTO UTXOs(OutPoint, Value, LockTime, AtHeight, ScriptHash) VALUES(?,?,?,?,?)" - _, err = db.Exec(sql, utxo.Op.Bytes(), valueBytes, utxo.LockTime, utxo.AtHeight, hash.Bytes()) - if err != nil { - return err - } - - return nil + sql := "INSERT OR REPLACE INTO UTXOs(OutPoint, Value, LockTime, AtHeight, Address) VALUES(?,?,?,?,?)" + _, err = u.Exec(sql, utxo.Op.Bytes(), valueBytes, utxo.LockTime, utxo.AtHeight, utxo.Address.Bytes()) + return err } // get a utxo from database -func (db *UTXOs) Get(outPoint *core.OutPoint) (*util.UTXO, error) { - db.RLock() - defer db.RUnlock() +func (u *utxos) Get(outPoint *core.OutPoint) (*sutil.UTXO, error) { + u.RLock() + defer u.RUnlock() - row := db.QueryRow(`SELECT Value, LockTime, AtHeight FROM UTXOs WHERE OutPoint=?`, outPoint.Bytes()) + row := u.QueryRow(`SELECT Value, LockTime, AtHeight, Address FROM UTXOs WHERE OutPoint=?`, outPoint.Bytes()) var valueBytes []byte var lockTime uint32 var atHeight uint32 - err := row.Scan(&valueBytes, &lockTime, &atHeight) + var addressBytes []byte + err := row.Scan(&valueBytes, &lockTime, &atHeight, &addressBytes) if err != nil { return nil, err } - var value *common.Fixed64 - value, err = common.Fixed64FromBytes(valueBytes) + value, err := common.Fixed64FromBytes(valueBytes) + if err != nil { + return nil, err + } + address, err := common.Uint168FromBytes(addressBytes) if err != nil { return nil, err } - return &util.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight}, nil + return &sutil.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address}, nil } // get utxos of the given script hash from database -func (db *UTXOs) GetAddrAll(hash *common.Uint168) ([]*util.UTXO, error) { - db.RLock() - defer db.RUnlock() +func (u *utxos) GetAddrAll(hash *common.Uint168) ([]*sutil.UTXO, error) { + u.RLock() + defer u.RUnlock() - rows, err := db.Query( - "SELECT OutPoint, Value, LockTime, AtHeight FROM UTXOs WHERE ScriptHash=?", hash.Bytes()) + rows, err := u.Query( + "SELECT OutPoint, Value, LockTime, AtHeight, Address FROM UTXOs WHERE ScriptHash=?", hash.Bytes()) if err != nil { return nil, err } defer rows.Close() - return db.getUTXOs(rows) + return u.getUTXOs(rows) } -func (db *UTXOs) GetAll() ([]*util.UTXO, error) { - db.RLock() - defer db.RUnlock() +func (u *utxos) GetAll() ([]*sutil.UTXO, error) { + u.RLock() + defer u.RUnlock() - rows, err := db.Query("SELECT OutPoint, Value, LockTime, AtHeight FROM UTXOs") + rows, err := u.Query("SELECT OutPoint, Value, LockTime, AtHeight, Address FROM UTXOs") if err != nil { - return []*util.UTXO{}, err + return []*sutil.UTXO{}, err } defer rows.Close() - return db.getUTXOs(rows) + return u.getUTXOs(rows) } -func (db *UTXOs) getUTXOs(rows *sql.Rows) ([]*util.UTXO, error) { - var utxos []*util.UTXO +func (u *utxos) getUTXOs(rows *sql.Rows) ([]*sutil.UTXO, error) { + var utxos []*sutil.UTXO for rows.Next() { var opBytes []byte var valueBytes []byte var lockTime uint32 var atHeight uint32 - err := rows.Scan(&opBytes, &valueBytes, &lockTime, &atHeight) + var addressBytes []byte + err := rows.Scan(&opBytes, &valueBytes, &lockTime, &atHeight, &addressBytes) if err != nil { return utxos, err } @@ -116,26 +120,41 @@ func (db *UTXOs) getUTXOs(rows *sql.Rows) ([]*util.UTXO, error) { if err != nil { return utxos, err } - var value *common.Fixed64 - value, err = common.Fixed64FromBytes(valueBytes) + value, err := common.Fixed64FromBytes(valueBytes) if err != nil { return utxos, err } - utxos = append(utxos, &util.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight}) + address, err := common.Uint168FromBytes(addressBytes) + if err != nil { + return utxos, err + } + utxo := &sutil.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address} + utxos = append(utxos, utxo) } return utxos, nil } // delete a utxo from database -func (db *UTXOs) Delete(outPoint *core.OutPoint) error { - db.Lock() - defer db.Unlock() +func (u *utxos) Del(outPoint *core.OutPoint) error { + u.Lock() + defer u.Unlock() + + _, err := u.Exec("DELETE FROM UTXOs WHERE OutPoint=?", outPoint.Bytes()) + return err +} + +func (u *utxos) Batch() UTXOsBatch { + u.Lock() + defer u.Unlock() - _, err := db.Exec("DELETE FROM UTXOs WHERE OutPoint=?", outPoint.Bytes()) + tx, err := u.DB.Begin() if err != nil { - return err + panic(err) } - return nil + return &utxosBatch{ + RWMutex: u.RWMutex, + Tx: tx, + } } diff --git a/spvwallet/store/sqlite/utxosbatch.go b/spvwallet/store/sqlite/utxosbatch.go new file mode 100644 index 0000000..fb6c242 --- /dev/null +++ b/spvwallet/store/sqlite/utxosbatch.go @@ -0,0 +1,41 @@ +package sqlite + +import ( + "database/sql" + "sync" + + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + + "github.com/elastos/Elastos.ELA/core" +) + +// Ensure utxos implement UTXOs interface. +var _ UTXOsBatch = (*utxosBatch)(nil) + +type utxosBatch struct { + *sync.RWMutex + *sql.Tx +} + +// put a utxo to database +func (db *utxosBatch) Put(utxo *sutil.UTXO) error { + db.Lock() + defer db.Unlock() + + valueBytes, err := utxo.Value.Bytes() + if err != nil { + return err + } + sql := "INSERT OR REPLACE INTO UTXOs(OutPoint, Value, LockTime, AtHeight, Address) VALUES(?,?,?,?,?)" + _, err = db.Exec(sql, utxo.Op.Bytes(), valueBytes, utxo.LockTime, utxo.AtHeight, utxo.Address.Bytes()) + return err +} + +// delete a utxo from database +func (db *utxosBatch) Del(outPoint *core.OutPoint) error { + db.Lock() + defer db.Unlock() + + _, err := db.Exec("DELETE FROM UTXOs WHERE OutPoint=?", outPoint.Bytes()) + return err +} From b4acce5dc01d6f56cf65eb6fe69d405cd29b9a31 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 11 Sep 2018 10:45:25 +0800 Subject: [PATCH 22/73] initial refactor commit for spvwallet/sutil/ package --- spvwallet/sutil/addr.go | 2 +- spvwallet/sutil/stxo.go | 11 ++++++++++- spvwallet/sutil/tx.go | 2 +- spvwallet/sutil/utxo.go | 34 ++++++++++++++++++++++++++-------- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/spvwallet/sutil/addr.go b/spvwallet/sutil/addr.go index 700e66d..a0645a5 100644 --- a/spvwallet/sutil/addr.go +++ b/spvwallet/sutil/addr.go @@ -1,4 +1,4 @@ -package util +package sutil import "github.com/elastos/Elastos.ELA.Utility/common" diff --git a/spvwallet/sutil/stxo.go b/spvwallet/sutil/stxo.go index 1f767a5..1ad66fb 100644 --- a/spvwallet/sutil/stxo.go +++ b/spvwallet/sutil/stxo.go @@ -1,4 +1,4 @@ -package util +package sutil import ( "fmt" @@ -27,6 +27,7 @@ func (stxo *STXO) String() string { "SendHeight:", stxo.SpendHeight, ",", "SpendTxId:", stxo.SpendTxId.String(), "}") } + func (stxo *STXO) IsEqual(alt *STXO) bool { if alt == nil { return stxo == nil @@ -46,3 +47,11 @@ func (stxo *STXO) IsEqual(alt *STXO) bool { return true } + +func NewSTXO(utxo *UTXO, spendHeight uint32, spendTxId common.Uint256) *STXO { + return &STXO{ + UTXO: *utxo, + SpendHeight: spendHeight, + SpendTxId: spendTxId, + } +} diff --git a/spvwallet/sutil/tx.go b/spvwallet/sutil/tx.go index 3008d32..6fb0c87 100644 --- a/spvwallet/sutil/tx.go +++ b/spvwallet/sutil/tx.go @@ -1,4 +1,4 @@ -package util +package sutil import ( "time" diff --git a/spvwallet/sutil/utxo.go b/spvwallet/sutil/utxo.go index b17fe3d..b37ee33 100644 --- a/spvwallet/sutil/utxo.go +++ b/spvwallet/sutil/utxo.go @@ -1,11 +1,11 @@ -package util +package sutil import ( "fmt" "sort" - "github.com/elastos/Elastos.ELA/core" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" ) type UTXO struct { @@ -20,6 +20,9 @@ type UTXO struct { // Block height where this tx was confirmed, 0 for unconfirmed AtHeight uint32 + + // Address where this UTXO belongs to. + Address common.Uint168 } func (utxo *UTXO) String() string { @@ -53,14 +56,29 @@ func (utxo *UTXO) IsEqual(alt *UTXO) bool { return false } + if !utxo.Address.IsEqual(alt.Address) { + return false + } + return true } -type SortableUTXOs []*UTXO +func NewUTXO(txId common.Uint256, height uint32, index int, + value common.Fixed64, lockTime uint32, address common.Uint168) *UTXO { + utxo := new(UTXO) + utxo.Op = *core.NewOutPoint(txId, uint16(index)) + utxo.Value = value + utxo.LockTime = lockTime + utxo.AtHeight = height + utxo.Address = address + return utxo +} + +type SortByValueASC []*UTXO -func (utxos SortableUTXOs) Len() int { return len(utxos) } -func (utxos SortableUTXOs) Swap(i, j int) { utxos[i], utxos[j] = utxos[j], utxos[i] } -func (utxos SortableUTXOs) Less(i, j int) bool { +func (utxos SortByValueASC) Len() int { return len(utxos) } +func (utxos SortByValueASC) Swap(i, j int) { utxos[i], utxos[j] = utxos[j], utxos[i] } +func (utxos SortByValueASC) Less(i, j int) bool { if utxos[i].Value > utxos[j].Value { return false } else { @@ -68,8 +86,8 @@ func (utxos SortableUTXOs) Less(i, j int) bool { } } -func SortUTXOs(utxos []*UTXO) []*UTXO { - sortableUTXOs := SortableUTXOs(utxos) +func SortByValue(utxos []*UTXO) []*UTXO { + sortableUTXOs := SortByValueASC(utxos) sort.Sort(sortableUTXOs) return sortableUTXOs } From 1a5bb91faeb06db5ba50fb5c77c6927801b5a786 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 11 Sep 2018 10:46:13 +0800 Subject: [PATCH 23/73] initial refactor commit for spvwallet/ package --- spvwallet/log.go | 28 +++ spvwallet/txidcache.go | 21 +- spvwallet/wallet.go | 472 +++++++++++++++++++++++++---------------- 3 files changed, 323 insertions(+), 198 deletions(-) create mode 100644 spvwallet/log.go diff --git a/spvwallet/log.go b/spvwallet/log.go new file mode 100644 index 0000000..13998e8 --- /dev/null +++ b/spvwallet/log.go @@ -0,0 +1,28 @@ +package spvwallet + +import ( + "github.com/elastos/Elastos.ELA.Utility/elalog" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log elalog.Logger + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = elalog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using elalog. +func UseLogger(logger elalog.Logger) { + log = logger +} diff --git a/spvwallet/txidcache.go b/spvwallet/txidcache.go index 249454e..748a89b 100644 --- a/spvwallet/txidcache.go +++ b/spvwallet/txidcache.go @@ -7,32 +7,28 @@ import ( type TxIdCache struct { sync.Mutex - txIds map[common.Uint256]uint32 + txIds map[common.Uint256]struct{} index uint32 txIdIndex []common.Uint256 } func NewTxIdCache(capacity int) *TxIdCache { return &TxIdCache{ - txIds: make(map[common.Uint256]uint32), + txIds: make(map[common.Uint256]struct{}), txIdIndex: make([]common.Uint256, capacity), } } -func (ic *TxIdCache) Add(txId common.Uint256, height uint32) bool { +func (ic *TxIdCache) Add(txId common.Uint256) bool { ic.Lock() defer ic.Unlock() - _, ok := ic.txIds[txId] - if ok { - return false - } // Remove oldest txId ic.index = ic.index % uint32(cap(ic.txIdIndex)) delete(ic.txIds, ic.txIdIndex[ic.index]) // Add new txId - ic.txIds[txId] = height + ic.txIds[txId] = struct{}{} ic.txIdIndex[ic.index] = txId // Increase index @@ -40,12 +36,9 @@ func (ic *TxIdCache) Add(txId common.Uint256, height uint32) bool { return true } -func (ic *TxIdCache) Get(txId common.Uint256) (uint32, bool) { +func (ic *TxIdCache) Get(txId common.Uint256) bool { ic.Lock() defer ic.Unlock() - height, ok := ic.txIds[txId] - if ok { - return height, ok - } - return 0, false + _, ok := ic.txIds[txId] + return ok } diff --git a/spvwallet/wallet.go b/spvwallet/wallet.go index 20b1f03..b3626bd 100644 --- a/spvwallet/wallet.go +++ b/spvwallet/wallet.go @@ -4,14 +4,15 @@ import ( "time" "github.com/elastos/Elastos.ELA.SPV/database" - "github.com/elastos/Elastos.ELA.SPV/log" "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/db" "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/headers" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/sqlite" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA/core" ) @@ -22,79 +23,126 @@ const ( MinPeersForSync = 2 ) -func Init(clientId uint64, seeds []string) (*SPVWallet, error) { - wallet := new(SPVWallet) +type Wallet struct { + sdk.IService + rpcServer *rpc.Server + chainStore database.ChainStore + db sqlite.DataStore + txIds *TxIdCache + filter *sdk.AddrFilter +} - // Initialize headers db - headers, err := db.NewHeadersDB() - if err != nil { - return nil, err +func (w *Wallet) Start() { + w.IService.Start() + w.rpcServer.Start() +} + +func (w *Wallet) Stop() { + w.IService.Stop() + w.rpcServer.Close() +} + +// Batch returns a TxBatch instance for transactions batch +// commit, this can get better performance when commit a bunch +// of transactions within a block. +func (w *Wallet) Batch() database.TxBatch { + return &txBatch{ + db: w.db, + batch: w.db.Batch(), + ids: w.txIds, + filter: w.getAddrFilter(), } +} - // Initialize wallet database - wallet.dataStore, err = db.NewSQLiteDB() - if err != nil { - return nil, err +// CommitTx save a transaction into database, and return +// if it is a false positive and error. +func (w *Wallet) CommitTx(tx *util.Tx) (bool, error) { + // In this SPV implementation, CommitTx only invoked on new transactions + // that are unconfirmed, we just check if this transaction is a false + // positive and store it into database. NOTICE: at this moment, we didn't + // change UTXOs and STXOs. + txId := tx.Hash() + + // We already have this transaction. + ok := w.txIds.Get(txId) + if ok { + return false, nil } - // Initiate ChainStore - wallet.chainStore = database.NewDefaultChainDB(headers, wallet) + hits := 0 + // Check if any UTXOs of this wallet are used. + for _, input := range tx.Inputs { + stxo, _ := w.db.UTXOs().Get(&input.Previous) + if stxo != nil { + hits++ + } + } - // Initialize txs cache - wallet.txIds = NewTxIdCache(MaxTxIdCached) + // Check if there are any output to this wallet address. + for _, output := range tx.Outputs { + if w.getAddrFilter().ContainAddr(output.ProgramHash) { + hits++ + } + } - // Initialize spv service - wallet.IService, err = sdk.NewService( - &sdk.Config{ - Magic: config.Values().Magic, - SeedList: config.Values().SeedList, - MaxPeers: MaxPeers, - MinPeersForSync: MinPeersForSync, - Foundation: config.Values().Foundation, - ChainStore: wallet.chainStore, - GetFilterData: wallet.GetFilterData, - }) + // If no hits, no need to save transaction + if hits == 0 { + return true, nil + } + + // Save transaction as unconfirmed. + err := w.db.Txs().Put(sutil.NewTx(tx.Transaction, 0)) if err != nil { - return nil, err + return false, err } - // Initialize RPC server - server := rpc.InitServer() - server.NotifyNewAddress = wallet.NotifyNewAddress - server.SendTransaction = wallet.IService.SendTransaction - wallet.rpcServer = server + w.txIds.Add(txId) - return wallet, nil + return false, nil } -type DataListener interface { - OnNewBlock(block *msg.MerkleBlock, txs []*core.Transaction) - OnRollback(height uint32) +// HaveTx returns if the transaction already saved in database +// by it's id. +func (w *Wallet) HaveTx(txId *common.Uint256) (bool, error) { + tx, err := w.db.Txs().Get(txId) + return tx != nil, err } -type SPVWallet struct { - sdk.IService - rpcServer *rpc.Server - chainStore database.ChainStore - dataStore db.DataStore - txIds *TxIdCache - filter *sdk.AddrFilter +// GetTxs returns all transactions within the given height. +func (w *Wallet) GetTxs(height uint32) ([]*util.Tx, error) { + return nil, nil } -func (wallet *SPVWallet) Start() { - wallet.IService.Start() - wallet.rpcServer.Start() +// RemoveTxs delete all transactions on the given height. Return +// how many transactions are deleted from database. +func (w *Wallet) RemoveTxs(height uint32) (int, error) { + batch := w.db.Batch() + err := batch.RollbackHeight(height) + if err != nil { + return 0, batch.Rollback() + } + return 0, batch.Commit() } -func (wallet *SPVWallet) Stop() { - wallet.IService.Stop() - wallet.rpcServer.Close() +// Clear delete all data in database. +func (w *Wallet) Clear() error { + return w.db.Clear() } -func (wallet *SPVWallet) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { - utxos, _ := wallet.dataStore.UTXOs().GetAll() - stxos, _ := wallet.dataStore.STXOs().GetAll() +// Close database. +func (w *Wallet) Close() error { + return w.db.Close() +} +func (w *Wallet) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { + utxos, err := w.db.UTXOs().GetAll() + if err != nil { + log.Debugf("GetAll UTXOs error: %v", err) + } + stxos, err := w.db.STXOs().GetAll() + if err != nil { + log.Debugf("GetAll STXOs error: %v", err) + } outpoints := make([]*core.OutPoint, 0, len(utxos)+len(stxos)) for _, utxo := range utxos { outpoints = append(outpoints, &utxo.Op) @@ -103,84 +151,135 @@ func (wallet *SPVWallet) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { outpoints = append(outpoints, &stxo.Op) } - return wallet.getAddrFilter().GetAddrs(), outpoints + return w.getAddrFilter().GetAddrs(), outpoints } -func (wallet *SPVWallet) isFalsePositive(tx *core.Transaction) (bool, error) { +func (w *Wallet) NotifyNewAddress(hash []byte) { + // Reload address filter to include new address + w.loadAddrFilter() + // Broadcast filterload message to connected peers + w.UpdateFilter() +} - hits := 0 - // Save UTXOs - for index, output := range tx.Outputs { - // Filter address - if wallet.getAddrFilter().ContainAddr(output.ProgramHash) { - var lockTime uint32 - if tx.TxType == core.CoinBase { - lockTime = height + 100 - } - utxo := ToUTXO(txId, height, index, output.Value, lockTime) - err := wallet.dataStore.UTXOs().Put(&output.ProgramHash, utxo) - if err != nil { - return false, err - } - hits++ - } +func (w *Wallet) getAddrFilter() *sdk.AddrFilter { + if w.filter == nil { + w.loadAddrFilter() } + return w.filter +} - // Put spent UTXOs to STXOs - for _, input := range tx.Inputs { - // Try to move UTXO to STXO, if a UTXO in database was spent, it will be moved to STXO - err := wallet.dataStore.STXOs().FromUTXO(&input.Previous, &txId, height) - if err == nil { - hits++ - } +func (w *Wallet) loadAddrFilter() *sdk.AddrFilter { + addrs, _ := w.db.Addrs().GetAll() + w.filter = sdk.NewAddrFilter(nil) + for _, addr := range addrs { + w.filter.AddAddr(addr.Hash()) } + return w.filter +} - // If no hits, no need to save transaction - if hits == 0 { - return true, nil - } +// TransactionAccepted will be invoked after a transaction sent by +// SendTransaction() method has been accepted. Notice: this method needs at +// lest two connected peers to work. +func (w *Wallet) TransactionAccepted(tx *util.Tx) { + // TODO } -// Commit a transaction return if this is a false positive and error -func (wallet *SPVWallet) CommitTx(tx *core.Transaction, height uint32) (bool, error) { - txId := tx.Hash() +// TransactionRejected will be invoked if a transaction sent by SendTransaction() +// method has been rejected. +func (w *Wallet) TransactionRejected(tx *util.Tx) { + // TODO - sh, ok := wallet.txIds.Get(txId) - if ok && (sh > 0 || (sh == 0 && height == 0)) { - return false, nil - } +} + +// TransactionConfirmed will be invoked after a transaction sent by +// SendTransaction() method has been packed into a block. +func (w *Wallet) TransactionConfirmed(tx *util.Tx) { + // TODO - // Do not check double spends when syncing - if wallet.IsCurrent() { - dubs, err := wallet.checkDoubleSpends(tx) +} + +// BlockCommitted will be invoked when a block and transactions within it are +// successfully committed into database. +func (w *Wallet) BlockCommitted(block *util.Block) { + if w.IsCurrent() { + w.db.State().PutHeight(block.Height) + // Get all unconfirmed transactions + txs, err := w.db.Txs().GetAllUnconfirmed() if err != nil { - return false, nil + log.Debugf("Get unconfirmed transactions failed, error %s", err.Error()) + return } - if len(dubs) > 0 { - if height == 0 { - return false, nil - } else { - // Rollback any double spend transactions - for _, dub := range dubs { - if err := wallet.dataStore.RollbackTx(dub); err != nil { - return false, nil - } + now := time.Now() + for _, tx := range txs { + if now.After(tx.Timestamp.Add(MaxUnconfirmedTime)) { + err = w.db.Txs().Del(&tx.TxId) + if err != nil { + log.Errorf("Delete timeout transaction %s failed, error %s", tx.TxId.String(), err.Error()) } } } } +} + +type txBatch struct { + db sqlite.DataStore + batch sqlite.DataBatch + ids *TxIdCache + filter *sdk.AddrFilter +} + +// AddTx add a store transaction operation into batch, and return +// if it is a false positive and error. +func (b *txBatch) AddTx(tx *util.Tx) (bool, error) { + // This AddTx in batch used by the ChainStore when storing a block. + // That means this transaction has been confirmed, so we need to remove + // it from unconfirmed list. And also, double spend transactions should + // been removed from unconfirmed list as well. + txId := tx.Hash() + height := tx.Height + dubs, err := b.checkDoubleSpends(tx) + if err != nil { + return false, nil + } + // Delete any double spend transactions + if len(dubs) > 0 { + batch := b.db.Txs().Batch() + for _, dub := range dubs { + if err := batch.Del(dub); err != nil { + batch.Rollback() + return false, nil + } + } + batch.Commit() + } hits := 0 - // Save UTXOs + // Check if any UTXOs within this wallet have been spent. + for _, input := range tx.Inputs { + // Move UTXO to STXO + utxo, _ := b.db.UTXOs().Get(&input.Previous) + // Skip if no match. + if utxo == nil { + continue + } + + err := b.batch.STXOs().Put(sutil.NewSTXO(utxo, height, txId)) + if err != nil { + return false, nil + } + hits++ + } + + // Check if there are any output to this wallet address. for index, output := range tx.Outputs { // Filter address - if wallet.getAddrFilter().ContainAddr(output.ProgramHash) { + if b.filter.ContainAddr(output.ProgramHash) { var lockTime uint32 if tx.TxType == core.CoinBase { lockTime = height + 100 } - utxo := ToUTXO(txId, height, index, output.Value, lockTime) - err := wallet.dataStore.UTXOs().Put(&output.ProgramHash, utxo) + utxo := sutil.NewUTXO(txId, height, index, output.Value, lockTime, output.ProgramHash) + err := b.batch.UTXOs().Put(utxo) if err != nil { return false, err } @@ -188,120 +287,125 @@ func (wallet *SPVWallet) CommitTx(tx *core.Transaction, height uint32) (bool, er } } - // Put spent UTXOs to STXOs - for _, input := range tx.Inputs { - // Try to move UTXO to STXO, if a UTXO in database was spent, it will be moved to STXO - err := wallet.dataStore.STXOs().FromUTXO(&input.Previous, &txId, height) - if err == nil { - hits++ - } - } - // If no hits, no need to save transaction if hits == 0 { return true, nil } // Save transaction - err := wallet.dataStore.Txs().Put(db.NewTx(*tx, height)) + err = b.batch.Txs().Put(sutil.NewTx(tx.Transaction, height)) if err != nil { return false, err } - wallet.txIds.Add(txId, height) + b.ids.Add(txId) return false, nil } -func (wallet *SPVWallet) OnBlockCommitted(block *msg.MerkleBlock, txs []*core.Transaction) { - wallet.dataStore.Chain().PutHeight(block.Header.(*core.Header).Height) - - // Check unconfirmed transaction timeout - if wallet.IsCurrent() { - // Get all unconfirmed transactions - txs, err := wallet.dataStore.Txs().GetAllFrom(0) - if err != nil { - log.Debugf("Get unconfirmed transactions failed, error %s", err.Error()) - return - } - now := time.Now() - for _, tx := range txs { - if now.After(tx.Timestamp.Add(MaxUnconfirmedTime)) { - err = wallet.dataStore.RollbackTx(&tx.TxId) - if err != nil { - log.Errorf("Rollback timeout transaction %s failed, error %s", tx.TxId.String(), err.Error()) - } - } - } - } -} - -// Rollback chain data on the given height -func (wallet *SPVWallet) OnRollback(height uint32) error { - return wallet.dataStore.Rollback(height) -} - -func ToUTXO(txId common.Uint256, height uint32, index int, value common.Fixed64, lockTime uint32) *db.UTXO { - utxo := new(db.UTXO) - utxo.Op = *core.NewOutPoint(txId, uint16(index)) - utxo.Value = value - utxo.LockTime = lockTime - utxo.AtHeight = height - return utxo +// DelTx add a delete transaction operation into batch. +func (b *txBatch) DelTx(txId *common.Uint256) error { + return b.batch.Txs().Del(txId) } -func (wallet *SPVWallet) NotifyNewAddress(hash []byte) { - // Reload address filter to include new address - wallet.loadAddrFilter() - // Broadcast filterload message to connected peers - wallet.UpdateFilter() +// DelTxs add a delete transactions on given height operation. +func (b *txBatch) DelTxs(height uint32) error { + // Delete transactions is used when blockchain doing rollback, this not + // only delete the transactions on the given height, and also restore + // STXOs and remove UTXOs within these transactions. + return b.batch.RollbackHeight(height) } -func (wallet *SPVWallet) getAddrFilter() *sdk.AddrFilter { - if wallet.filter == nil { - wallet.loadAddrFilter() - } - return wallet.filter +// Rollback cancel all operations in current batch. +func (b *txBatch) Rollback() error { + return b.batch.Rollback() } -func (wallet *SPVWallet) loadAddrFilter() *sdk.AddrFilter { - addrs, _ := wallet.dataStore.Addrs().GetAll() - wallet.filter = sdk.NewAddrFilter(nil) - for _, addr := range addrs { - wallet.filter.AddAddr(addr.Hash()) - } - return wallet.filter +// Commit the added transactions into database. +func (b *txBatch) Commit() error { + return b.batch.Commit() } -// checkDoubleSpends takes a transaction and compares it with -// all transactions in the db. It returns a slice of all txIds in the db -// which are double spent by the received tx. -func (wallet *SPVWallet) checkDoubleSpends(tx *core.Transaction) ([]*common.Uint256, error) { - var dubs []*common.Uint256 +// checkDoubleSpends takes a transaction and compares it with all unconfirmed +// transactions in the db. It returns a slice of txIds in the db which are +// double spent by the received tx. +func (b *txBatch) checkDoubleSpends(tx *util.Tx) ([]*common.Uint256, error) { txId := tx.Hash() - txs, err := wallet.dataStore.Txs().GetAll() + txs, err := b.db.Txs().GetAllUnconfirmed() if err != nil { return nil, err } + + inputs := make(map[string]*common.Uint256) for _, compTx := range txs { // Skip coinbase transaction if compTx.Data.IsCoinBaseTx() { continue } + // Skip duplicate transaction compTxId := compTx.Data.Hash() if compTxId.IsEqual(txId) { continue } - for _, txIn := range tx.Inputs { - for _, compIn := range compTx.Data.Inputs { - if txIn.Previous.IsEqual(compIn.Previous) { - // Found double spend - dubs = append(dubs, &compTxId) - break // back to txIn loop - } - } + + for _, in := range compTx.Data.Inputs { + inputs[in.ReferKey()] = &compTxId + } + } + + var dubs []*common.Uint256 + for _, in := range tx.Inputs { + if tx, ok := inputs[in.ReferKey()]; ok { + dubs = append(dubs, tx) } } return dubs, nil } + +func New() (*Wallet, error) { + wallet := new(Wallet) + + // Initialize headers db + headers, err := headers.New() + if err != nil { + return nil, err + } + + // Initialize singleton database + wallet.db, err = sqlite.New() + if err != nil { + return nil, err + } + + // Initiate ChainStore + wallet.chainStore = database.NewDefaultChainDB(headers, wallet) + + // Initialize txs cache + wallet.txIds = NewTxIdCache(MaxTxIdCached) + + // Initialize spv service + wallet.IService, err = sdk.NewService( + &sdk.Config{ + Magic: config.Values().Magic, + SeedList: config.Values().SeedList, + DefaultPort: config.Values().DefaultPort, + MaxPeers: MaxPeers, + MinPeersForSync: MinPeersForSync, + Foundation: config.Values().Foundation, + ChainStore: wallet.chainStore, + GetFilterData: wallet.GetFilterData, + StateNotifier: wallet, + }) + if err != nil { + return nil, err + } + + // Initialize RPC server + server := rpc.InitServer() + server.NotifyNewAddress = wallet.NotifyNewAddress + server.SendTransaction = wallet.IService.SendTransaction + wallet.rpcServer = server + + return wallet, nil +} From 9c6c99973e27cbde5afe907426cfafb98e6e365a Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 11 Sep 2018 10:50:06 +0800 Subject: [PATCH 24/73] initial refactor commit on client.go --- client.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/client.go b/client.go index 0fc7f19..27a7f55 100644 --- a/client.go +++ b/client.go @@ -3,25 +3,15 @@ package main import ( "os" - "github.com/elastos/Elastos.ELA.SPV/log" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/cli/account" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/cli/transaction" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/cli/wallet" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/client/account" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/client/transaction" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/client/wallet" "github.com/urfave/cli" ) var Version string -func init() { - log.Init( - config.Values().PrintLevel, - config.Values().MaxPerLogSize, - config.Values().MaxLogsSize, - ) -} - func main() { app := cli.NewApp() app.Name = "ELASTOS SPV WALLET" From 7f0b69e564f2972d2a29f32e3e3125246ebdd176 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 11 Sep 2018 13:59:14 +0800 Subject: [PATCH 25/73] implement Serializable interface for util/tx.go --- util/block.go | 2 +- util/tx.go | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/util/block.go b/util/block.go index 650a931..6beea28 100644 --- a/util/block.go +++ b/util/block.go @@ -1,6 +1,6 @@ package util -// Block represent a block that stored into +// Block represent a block that stored in the // blockchain database. type Block struct { // header of this block. diff --git a/util/tx.go b/util/tx.go index 94e11e6..e959753 100644 --- a/util/tx.go +++ b/util/tx.go @@ -1,6 +1,11 @@ package util -import "github.com/elastos/Elastos.ELA/core" +import ( + "encoding/binary" + "io" + + "github.com/elastos/Elastos.ELA/core" +) // Tx is a data structure used in database. type Tx struct { @@ -10,4 +15,25 @@ type Tx struct { // The block height that this transaction // belongs to. Height uint32 -} \ No newline at end of file +} + +func NewTx(tx core.Transaction, height uint32) *Tx { + return &Tx{ + Transaction: tx, + Height: height, + } +} + +func (t *Tx) Serialize(buf io.Writer) error { + if err := t.Transaction.Serialize(buf); err != nil { + return err + } + return binary.Write(buf, binary.LittleEndian, t.Height) +} + +func (t *Tx) Deserialize(reader io.Reader) error { + if err := t.Transaction.Deserialize(reader); err != nil { + return err + } + return binary.Read(reader, binary.LittleEndian, &t.Height) +} From 46f82d3624ae425889b1fbd82a8064b71a6f26d8 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Wed, 12 Sep 2018 10:38:02 +0800 Subject: [PATCH 26/73] initial refactor commit for main.go --- main.go | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index edba08a..f8409fc 100644 --- a/main.go +++ b/main.go @@ -3,32 +3,40 @@ package main import ( "os" "os/signal" - "encoding/binary" + "github.com/elastos/Elastos.ELA.SPV/blockchain" + "github.com/elastos/Elastos.ELA.SPV/log" + "github.com/elastos/Elastos.ELA.SPV/peer" + "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/spvwallet" "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" - "github.com/elastos/Elastos.ELA.SPV/log" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/store" + "github.com/elastos/Elastos.ELA.SPV/sync" ) +const LogPath = "./Logs-spv/" + func main() { - // Initiate log - log.Init( + // Initiate logger + logger := log.NewLogger(LogPath, config.Values().PrintLevel, config.Values().MaxPerLogSize, config.Values().MaxLogsSize, ) - file, err := spvwallet.OpenKeystoreFile() - if err != nil { - log.Error("Keystore.dat file not found, please create your wallet using ela-wallet first") - os.Exit(0) - } + sdk.UseLogger(logger) + rpc.UseLogger(logger) + peer.UseLogger(logger) + blockchain.UseLogger(logger) + store.UseLogger(logger) + sync.UseLogger(logger) + spvwallet.UseLogger(logger) // Initiate SPV service - iv, _ := file.GetIV() - wallet, err := spvwallet.Init(binary.LittleEndian.Uint64(iv), config.Values().SeedList) + wallet, err := spvwallet.New() if err != nil { - log.Error("Initiate SPV service failed,", err) + logger.Error("Initiate SPV service failed,", err) os.Exit(0) } @@ -38,7 +46,7 @@ func main() { signal.Notify(c, os.Interrupt) go func() { for range c { - log.Trace("SPVWallet shutting down...") + logger.Trace("Wallet shutting down...") wallet.Stop() stop <- 1 } From 36dd94f1b5569368ecd913bbbd230d66c31593f6 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Wed, 12 Sep 2018 11:24:06 +0800 Subject: [PATCH 27/73] initial refactor commit for interface/store/ package --- interface/store/addrs.go | 101 +++++++++++++ interface/store/databatch.go | 102 +++++++++++++ interface/store/datastore.go | 109 ++++++++++++++ interface/store/headers.go | 267 +++++++++++++++++++++++++++++++++++ interface/store/interface.go | 112 +++++++++++++++ interface/store/ops.go | 100 +++++++++++++ interface/store/opsbatch.go | 29 ++++ interface/store/que.go | 120 ++++++++++++++++ interface/store/quebatch.go | 44 ++++++ interface/store/txs.go | 200 ++++++++++++++++++++++++++ interface/store/txsbatch.go | 165 ++++++++++++++++++++++ 11 files changed, 1349 insertions(+) create mode 100644 interface/store/addrs.go create mode 100644 interface/store/databatch.go create mode 100644 interface/store/datastore.go create mode 100644 interface/store/headers.go create mode 100644 interface/store/interface.go create mode 100644 interface/store/ops.go create mode 100644 interface/store/opsbatch.go create mode 100644 interface/store/que.go create mode 100644 interface/store/quebatch.go create mode 100644 interface/store/txs.go create mode 100644 interface/store/txsbatch.go diff --git a/interface/store/addrs.go b/interface/store/addrs.go new file mode 100644 index 0000000..5cce77e --- /dev/null +++ b/interface/store/addrs.go @@ -0,0 +1,101 @@ +package store + +import ( + "sync" + + "github.com/elastos/Elastos.ELA.SPV/sdk" + + "github.com/elastos/Elastos.ELA.Utility/common" + + "github.com/boltdb/bolt" +) + +var ( + BKTAddrs = []byte("Addrs") +) + +// Ensure addrs implement Addrs interface. +var _ Addrs = (*addrs)(nil) + +type addrs struct { + *sync.RWMutex + *bolt.DB + filter *sdk.AddrFilter +} + +func NewAddrs(db *bolt.DB) (*addrs, error) { + store := new(addrs) + store.RWMutex = new(sync.RWMutex) + store.DB = db + + db.Update(func(btx *bolt.Tx) error { + _, err := btx.CreateBucketIfNotExists(BKTAddrs) + if err != nil { + return err + } + return nil + }) + + addrs, err := store.getAll() + if err != nil { + return nil, err + } + store.filter = sdk.NewAddrFilter(addrs) + + return store, nil +} + +func (a *addrs) GetFilter() *sdk.AddrFilter { + a.Lock() + defer a.Unlock() + return a.filter +} + +func (a *addrs) Put(addr *common.Uint168) error { + a.Lock() + defer a.Unlock() + + if a.filter.ContainAddr(*addr) { + return nil + } + + a.filter.AddAddr(addr) + return a.Update(func(tx *bolt.Tx) error { + return tx.Bucket(BKTAddrs).Put(addr.Bytes(), addr.Bytes()) + }) +} + +func (a *addrs) GetAll() []*common.Uint168 { + a.RLock() + defer a.RUnlock() + + return a.filter.GetAddrs() +} + +func (a *addrs) getAll() (addrs []*common.Uint168, err error) { + err = a.View(func(tx *bolt.Tx) error { + return tx.Bucket(BKTAddrs).ForEach(func(k, v []byte) error { + addr, err := common.Uint168FromBytes(v) + if err != nil { + return err + } + addrs = append(addrs, addr) + return nil + }) + }) + + return addrs, err +} + +func (a *addrs) Clear() error { + a.Lock() + defer a.Unlock() + return a.DB.Update(func(tx *bolt.Tx) error { + return tx.DeleteBucket(BKTAddrs) + }) +} + +func (a *addrs) Close() error { + a.Lock() + return nil +} diff --git a/interface/store/databatch.go b/interface/store/databatch.go new file mode 100644 index 0000000..5436824 --- /dev/null +++ b/interface/store/databatch.go @@ -0,0 +1,102 @@ +package store + +import ( + "bytes" + "database/sql" + "encoding/binary" + "encoding/gob" + "github.com/boltdb/bolt" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" + "sync" +) + +// Ensure dataBatch implement DataBatch interface. +var _ DataBatch = (*dataBatch)(nil) + +type dataBatch struct { + mutex sync.Mutex + boltTx *bolt.Tx + sqlTx *sql.Tx +} + +func (b *dataBatch) Txs() TxsBatch { + return &txsBatch{Tx: b.boltTx} +} + +func (b *dataBatch) Ops() OpsBatch { + return &opsBatch{Tx: b.boltTx} +} + +func (b *dataBatch) Que() QueBatch { + return &queBatch{Tx: b.sqlTx} +} + +// Delete all transactions, ops, queued items on +// the given height. +func (b *dataBatch) DelAll(height uint32) error { + b.mutex.Lock() + defer b.mutex.Unlock() + + var key [4]byte + binary.LittleEndian.PutUint32(key[:], height) + data := b.boltTx.Bucket(BKTHeightTxs).Get(key[:]) + + var txMap = make(map[common.Uint256]uint32) + err := gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) + if err != nil { + return err + } + + txsBucket := b.boltTx.Bucket(BKTTxs) + opsBucket := b.boltTx.Bucket(BKTOps) + for txId := range txMap { + var txn util.Tx + data := txsBucket.Get(txId.Bytes()) + err := txn.Deserialize(bytes.NewReader(data)) + if err != nil { + return err + } + + for index := range txn.Outputs { + outpoint := core.NewOutPoint(txn.Hash(), uint16(index)).Bytes() + opsBucket.Delete(outpoint) + } + + if err := b.boltTx.Bucket(BKTTxs).Delete(txId.Bytes()); err != nil { + return err + } + } + + err = b.boltTx.Bucket(BKTHeightTxs).Delete(key[:]) + if err != nil { + return err + } + + return b.Que().DelAll(height) +} + +func (b *dataBatch) Commit() error { + if err := b.boltTx.Commit(); err != nil { + return err + } + + if err := b.sqlTx.Commit(); err != nil { + return err + } + + return nil +} + +func (b *dataBatch) Rollback() error { + if err := b.boltTx.Rollback(); err != nil { + return err + } + + if err := b.sqlTx.Rollback(); err != nil { + return err + } + + return nil +} diff --git a/interface/store/datastore.go b/interface/store/datastore.go new file mode 100644 index 0000000..6f81a6e --- /dev/null +++ b/interface/store/datastore.go @@ -0,0 +1,109 @@ +package store + +import ( + "github.com/boltdb/bolt" + "sync" +) + +// Ensure dataStore implement DataStore interface. +var _ DataStore = (*dataStore)(nil) + +type dataStore struct { + *sync.RWMutex + *bolt.DB + addrs *addrs + txs *txs + ops *ops + que *que +} + +func NewDataStore() (*dataStore, error) { + db, err := bolt.Open("data_store.bin", 0644, &bolt.Options{InitialMmapSize: 5000000}) + if err != nil { + return nil, err + } + store := new(dataStore) + store.RWMutex = new(sync.RWMutex) + store.DB = db + + store.addrs, err = NewAddrs(db) + if err != nil { + return nil, err + } + + store.txs, err = NewTxs(db) + if err != nil { + return nil, err + } + + store.ops, err = NewOps(db) + if err != nil { + return nil, err + } + + store.que, err = NewQue() + if err != nil { + return nil, err + } + + return store, nil +} + +func (d *dataStore) Addrs() Addrs { + return d.addrs +} + +func (d *dataStore) Txs() Txs { + return d.txs +} + +func (d *dataStore) Ops() Ops { + return d.ops +} + +func (d *dataStore) Que() Que { + return d.que +} + +func (d *dataStore) Batch() DataBatch { + boltTx, err := d.DB.Begin(true) + if err != nil { + panic(err) + } + + sqlTx, err := d.que.Begin() + if err != nil { + panic(err) + } + + return &dataBatch{ + boltTx: boltTx, + sqlTx: sqlTx, + } +} + +func (d *dataStore) Clear() error { + d.Lock() + defer d.Unlock() + + return d.Update(func(tx *bolt.Tx) error { + tx.DeleteBucket(BKTAddrs) + tx.DeleteBucket(BKTTxs) + tx.DeleteBucket(BKTHeightTxs) + tx.DeleteBucket(BKTOps) + return nil + }) +} + +// Close db +func (d *dataStore) Close() error { + d.addrs.Close() + d.txs.Close() + d.ops.Close() + + d.Lock() + defer d.Unlock() + + d.que.Close() + return d.DB.Close() +} diff --git a/interface/store/headers.go b/interface/store/headers.go new file mode 100644 index 0000000..6f44efd --- /dev/null +++ b/interface/store/headers.go @@ -0,0 +1,267 @@ +package store + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "math/big" + "sync" + + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/boltdb/bolt" + "github.com/cevaris/ordered_map" + "github.com/elastos/Elastos.ELA.Utility/common" +) + +var ( + BKTHeaders = []byte("Headers") + BKTHeightHash = []byte("HeightHash") + BKTChainTip = []byte("ChainTip") + KEYChainTip = []byte("ChainTip") +) + +// Ensure headers implement database.Headers interface. +var _ HeaderStore = (*headers)(nil) + +type headers struct { + *sync.RWMutex + *bolt.DB + cache *cache +} + +func NewHeaderStore() (*headers, error) { + db, err := bolt.Open("headers.bin", 0644, &bolt.Options{InitialMmapSize: 5000000}) + if err != nil { + return nil, err + } + + db.Update(func(btx *bolt.Tx) error { + _, err := btx.CreateBucketIfNotExists(BKTHeaders) + if err != nil { + return err + } + _, err = btx.CreateBucketIfNotExists(BKTHeightHash) + if err != nil { + return err + } + _, err = btx.CreateBucketIfNotExists(BKTChainTip) + if err != nil { + return err + } + return nil + }) + + headers := &headers{ + RWMutex: new(sync.RWMutex), + DB: db, + cache: newCache(100), + } + + headers.initCache() + + return headers, nil +} + +func (h *headers) initCache() { + best, err := h.GetBest() + if err != nil { + return + } + h.cache.tip = best + headers := []*util.Header{best} + for i := 0; i < 99; i++ { + sh, err := h.GetPrevious(best) + if err != nil { + break + } + headers = append(headers, sh) + } + for i := len(headers) - 1; i >= 0; i-- { + h.cache.Set(headers[i]) + } +} + +func (h *headers) Put(header *util.Header, newTip bool) error { + h.Lock() + defer h.Unlock() + + h.cache.Set(header) + if newTip { + h.cache.tip = header + } + return h.Update(func(tx *bolt.Tx) error { + + bytes, err := header.Serialize() + if err != nil { + return err + } + + err = tx.Bucket(BKTHeaders).Put(header.Hash().Bytes(), bytes) + if err != nil { + return err + } + + if newTip { + err = tx.Bucket(BKTChainTip).Put(KEYChainTip, bytes) + if err != nil { + return err + } + } + + var key [4]byte + binary.LittleEndian.PutUint32(key[:], header.Height) + return tx.Bucket(BKTHeightHash).Put(key[:], header.Hash().Bytes()) + }) +} + +func (h *headers) GetPrevious(header *util.Header) (*util.Header, error) { + if header.Height == 1 { + return &util.Header{TotalWork: new(big.Int)}, nil + } + return h.Get(&header.Previous) +} + +func (h *headers) Get(hash *common.Uint256) (header *util.Header, err error) { + h.RLock() + defer h.RUnlock() + + header, err = h.cache.Get(hash) + if err == nil { + return header, nil + } + + err = h.View(func(tx *bolt.Tx) error { + + header, err = getHeader(tx, BKTHeaders, hash.Bytes()) + if err != nil { + return err + } + + return nil + }) + + return header, err +} + +func (h *headers) GetBest() (header *util.Header, err error) { + h.RLock() + defer h.RUnlock() + + if h.cache.tip != nil { + return h.cache.tip, nil + } + + err = h.View(func(tx *bolt.Tx) error { + + header, err = getHeader(tx, BKTChainTip, KEYChainTip) + if err != nil { + return err + } + + return nil + }) + + return header, err +} + +func (h *headers) GetByHeight(height uint32) (header *util.Header, err error) { + h.RLock() + defer h.RUnlock() + + err = h.View(func(tx *bolt.Tx) error { + var key [4]byte + binary.LittleEndian.PutUint32(key[:], height) + hashBytes := tx.Bucket(BKTHeightHash).Get(key[:]) + header, err = getHeader(tx, BKTHeaders, hashBytes) + if err != nil { + return err + } + return err + }) + if err != nil { + return header, fmt.Errorf("header not exist on height %d", height) + } + + return header, err +} + +func (h *headers) Clear() error { + h.Lock() + defer h.Unlock() + + return h.Update(func(tx *bolt.Tx) error { + err := tx.DeleteBucket(BKTHeaders) + if err != nil { + return err + } + + return tx.DeleteBucket(BKTChainTip) + }) +} + +// Close db +func (h *headers) Close() error { + h.Lock() + defer h.Unlock() + return h.DB.Close() +} + +func getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*util.Header, error) { + headerBytes := tx.Bucket(bucket).Get(key) + if headerBytes == nil { + return nil, fmt.Errorf("header %s does not exist in database", hex.EncodeToString(key)) + } + + var header util.Header + err := header.Deserialize(headerBytes) + if err != nil { + return nil, err + } + + return &header, nil +} + +type cache struct { + sync.RWMutex + size int + tip *util.Header + headers *ordered_map.OrderedMap +} + +func newCache(size int) *cache { + return &cache{ + size: size, + headers: ordered_map.NewOrderedMap(), + } +} + +func (cache *cache) pop() { + iter := cache.headers.IterFunc() + k, ok := iter() + if ok { + cache.headers.Delete(k.Key) + } +} + +func (cache *cache) Set(header *util.Header) { + cache.Lock() + defer cache.Unlock() + + if cache.headers.Len() > cache.size { + cache.pop() + } + cache.headers.Set(header.Hash().String(), header) +} + +func (cache *cache) Get(hash *common.Uint256) (*util.Header, error) { + cache.RLock() + defer cache.RUnlock() + + sh, ok := cache.headers.Get(hash.String()) + if !ok { + return nil, errors.New("Header not found in cache ") + } + return sh.(*util.Header), nil +} diff --git a/interface/store/interface.go b/interface/store/interface.go new file mode 100644 index 0000000..66f9d59 --- /dev/null +++ b/interface/store/interface.go @@ -0,0 +1,112 @@ +package store + +import ( + "github.com/elastos/Elastos.ELA.SPV/database" + "github.com/elastos/Elastos.ELA.SPV/sdk" + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" +) + +type HeaderStore interface { + database.Headers + GetByHeight(height uint32) (header *util.Header, err error) +} + +type DataStore interface { + database.DB + Addrs() Addrs + Txs() Txs + Ops() Ops + Que() Que + Batch() DataBatch +} + +type DataBatch interface { + batch + Txs() TxsBatch + Ops() OpsBatch + Que() QueBatch + // Delete all transactions, ops, queued items on + // the given height. + DelAll(height uint32) error +} + +type batch interface { + Rollback() error + Commit() error +} + +type Addrs interface { + database.DB + GetFilter() *sdk.AddrFilter + Put(addr *common.Uint168) error + GetAll() []*common.Uint168 +} + +type Txs interface { + database.DB + Put(tx *util.Tx) error + Get(txId *common.Uint256) (*util.Tx, error) + GetAll() ([]*util.Tx, error) + GetIds(height uint32) ([]*common.Uint256, error) + Del(txId *common.Uint256) error + Batch() TxsBatch +} + +type TxsBatch interface { + batch + Put(tx *util.Tx) error + Del(txId *common.Uint256) error + DelAll(height uint32) error +} + +type Ops interface { + database.DB + Put(*core.OutPoint, common.Uint168) error + IsExist(*core.OutPoint) *common.Uint168 + GetAll() ([]*core.OutPoint, error) + Batch() OpsBatch +} + +type OpsBatch interface { + batch + Put(*core.OutPoint, common.Uint168) error + Del(*core.OutPoint) error +} + +type Que interface { + database.DB + + // Put a queue item to database + Put(item *QueItem) error + + // Get all items in queue + GetAll() ([]*QueItem, error) + + // Delete confirmed item in queue + Del(notifyId, txHash *common.Uint256) error + + // Batch returns a queue batch instance. + Batch() QueBatch +} + +type QueBatch interface { + batch + + // Put a queue item to database + Put(item *QueItem) error + + // Delete confirmed item in queue + Del(notifyId, txHash *common.Uint256) error + + // Delete all items on the given height. + DelAll(height uint32) error +} + +type QueItem struct { + NotifyId common.Uint256 + TxId common.Uint256 + Height uint32 +} diff --git a/interface/store/ops.go b/interface/store/ops.go new file mode 100644 index 0000000..ccf568b --- /dev/null +++ b/interface/store/ops.go @@ -0,0 +1,100 @@ +package store + +import ( + "sync" + + "github.com/boltdb/bolt" + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" +) + +var ( + BKTOps = []byte("Ops") +) + +// Ensure ops implement Ops interface. +var _ Ops = (*ops)(nil) + +type ops struct { + *sync.RWMutex + *bolt.DB +} + +func NewOps(db *bolt.DB) (*ops, error) { + store := new(ops) + store.RWMutex = new(sync.RWMutex) + store.DB = db + + db.Update(func(btx *bolt.Tx) error { + _, err := btx.CreateBucketIfNotExists(BKTOps) + if err != nil { + return err + } + return nil + }) + + return store, nil +} + +func (o *ops) Put(op *core.OutPoint, addr common.Uint168) (err error) { + o.Lock() + defer o.Unlock() + return o.Update(func(tx *bolt.Tx) error { + return tx.Bucket(BKTOps).Put(op.Bytes(), addr.Bytes()) + }) +} + +func (o *ops) IsExist(op *core.OutPoint) (addr *common.Uint168) { + o.RLock() + defer o.RUnlock() + + o.View(func(tx *bolt.Tx) error { + addrBytes := tx.Bucket(BKTOps).Get(op.Bytes()) + var err error + if addr, err = common.Uint168FromBytes(addrBytes); err != nil { + return err + } + return nil + }) + return addr +} + +func (o *ops) GetAll() (ops []*core.OutPoint, err error) { + o.RLock() + defer o.RUnlock() + + err = o.View(func(tx *bolt.Tx) error { + return tx.Bucket(BKTOps).ForEach(func(k, v []byte) error { + op, err := core.OutPointFromBytes(k) + if err != nil { + return err + } + ops = append(ops, op) + return nil + }) + }) + + return ops, err +} + +func (o *ops) Batch() OpsBatch { + tx, err := o.Begin(true) + if err != nil { + panic(err) + } + + return &opsBatch{Tx: tx} +} + +func (o *ops) Clear() error { + o.Lock() + defer o.Unlock() + return o.DB.Update(func(tx *bolt.Tx) error { + return tx.DeleteBucket(BKTOps) + }) +} + +func (o *ops) Close() error { + o.Lock() + return nil +} diff --git a/interface/store/opsbatch.go b/interface/store/opsbatch.go new file mode 100644 index 0000000..2a6bbc9 --- /dev/null +++ b/interface/store/opsbatch.go @@ -0,0 +1,29 @@ +package store + +import ( + "github.com/boltdb/bolt" + "sync" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" +) + +// Ensure opsBatch implement OpsBatch interface. +var _ OpsBatch = (*opsBatch)(nil) + +type opsBatch struct { + sync.Mutex + *bolt.Tx +} + +func (b *opsBatch) Put(op *core.OutPoint, addr common.Uint168) error { + b.Lock() + defer b.Unlock() + return b.Tx.Bucket(BKTOps).Put(op.Bytes(), addr.Bytes()) +} + +func (b *opsBatch) Del(op *core.OutPoint) error { + b.Lock() + defer b.Unlock() + return b.Tx.Bucket(BKTOps).Delete(op.Bytes()) +} diff --git a/interface/store/que.go b/interface/store/que.go new file mode 100644 index 0000000..dbce964 --- /dev/null +++ b/interface/store/que.go @@ -0,0 +1,120 @@ +package store + +import ( + "database/sql" + "fmt" + "sync" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +const ( + DriverName = "sqlite3" + DBName = "./queue.db" + + CreateQueueDB = `CREATE TABLE IF NOT EXISTS Queue( + NotifyId BLOB NOT NULL, + TxId BLOB NOT NULL, + Height INTEGER NOT NULL); + CREATE INDEX IF NOT EXISTS idx_queue_notify_id ON Queue (NotifyId); + CREATE INDEX IF NOT EXISTS idx_queue_tx_id ON Queue (TxId); + CREATE INDEX IF NOT EXISTS idx_queue_height ON Queue (height);` +) + +// Ensure que implement Que interface. +var _ Que = (*que)(nil) + +type que struct { + *sync.RWMutex + *sql.DB +} + +func NewQue() (*que, error) { + db, err := sql.Open(DriverName, DBName) + if err != nil { + fmt.Println("Open sqlite db error:", err) + return nil, err + } + + _, err = db.Exec(CreateQueueDB) + if err != nil { + return nil, err + } + return &que{RWMutex: new(sync.RWMutex), DB: db}, nil +} + +// Put a queue item to database +func (q *que) Put(item *QueItem) error { + q.Lock() + defer q.Unlock() + + sql := "INSERT OR REPLACE INTO Queue(NotifyId, TxId, Height) VALUES(?,?,?)" + _, err := q.Exec(sql, item.NotifyId.Bytes(), item.TxId.Bytes(), item.Height) + return err +} + +// Get all items in queue +func (q *que) GetAll() ([]*QueItem, error) { + q.RLock() + defer q.RUnlock() + + rows, err := q.Query("SELECT NotifyId, TxId, Height FROM Queue") + if err != nil { + return nil, err + } + + var items []*QueItem + for rows.Next() { + var notifyIdBytes []byte + var txHashBytes []byte + var height uint32 + err = rows.Scan(¬ifyIdBytes, &txHashBytes, &height) + if err != nil { + return nil, err + } + + notifyId, err := common.Uint256FromBytes(notifyIdBytes) + if err != nil { + return nil, err + } + txHash, err := common.Uint256FromBytes(txHashBytes) + if err != nil { + return nil, err + } + item := &QueItem{NotifyId: *notifyId, TxId: *txHash, Height: height} + items = append(items, item) + } + + return items, nil +} + +// Delete confirmed item in queue +func (q *que) Del(notifyId, txHash *common.Uint256) error { + q.Lock() + defer q.Unlock() + + _, err := q.Exec("DELETE FROM Queue WHERE NotifyId=? AND TxId=?", notifyId.Bytes(), txHash.Bytes()) + return err +} + +func (q *que) Batch() QueBatch { + tx, err := q.Begin() + if err != nil { + panic(err) + } + return &queBatch{Tx: tx} +} + +func (q *que) Clear() error { + q.Lock() + defer q.Unlock() + + _, err := q.Exec("DROP TABLE if EXISTS Queue") + return err +} + +func (q *que) Close() error { + q.Lock() + defer q.Unlock() + return q.DB.Close() +} diff --git a/interface/store/quebatch.go b/interface/store/quebatch.go new file mode 100644 index 0000000..f16c162 --- /dev/null +++ b/interface/store/quebatch.go @@ -0,0 +1,44 @@ +package store + +import ( + "database/sql" + "sync" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +// Ensure queBatch implement QueBatch interface. +var _ QueBatch = (*queBatch)(nil) + +type queBatch struct { + sync.Mutex + *sql.Tx +} + +// Put a queue item to database +func (b *queBatch) Put(item *QueItem) error { + b.Lock() + defer b.Unlock() + + sql := "INSERT OR REPLACE INTO Queue(NotifyId, TxId, Height) VALUES(?,?,?)" + _, err := b.Tx.Exec(sql, item.NotifyId.Bytes(), item.TxId.Bytes(), item.Height) + return err +} + +// Delete confirmed item in queue +func (b *queBatch) Del(notifyId, txHash *common.Uint256) error { + b.Lock() + defer b.Unlock() + + _, err := b.Tx.Exec("DELETE FROM Queue WHERE NotifyId=? AND TxId=?", notifyId.Bytes(), txHash.Bytes()) + return err +} + +// Delete all items on the given height. +func (b *queBatch) DelAll(height uint32) error { + b.Lock() + defer b.Unlock() + + _, err := b.Tx.Exec("DELETE FROM Queue WHERE Height=?", height) + return err +} diff --git a/interface/store/txs.go b/interface/store/txs.go new file mode 100644 index 0000000..249a68a --- /dev/null +++ b/interface/store/txs.go @@ -0,0 +1,200 @@ +package store + +import ( + "bytes" + "encoding/binary" + "encoding/gob" + "sync" + + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/boltdb/bolt" + "github.com/elastos/Elastos.ELA.Utility/common" +) + +var ( + BKTTxs = []byte("Txs") + BKTHeightTxs = []byte("HeightTxs") +) + +// Ensure txs implement Txs interface. +var _ Txs = (*txs)(nil) + +type txs struct { + *sync.RWMutex + *bolt.DB +} + +func NewTxs(db *bolt.DB) (*txs, error) { + store := new(txs) + store.RWMutex = new(sync.RWMutex) + store.DB = db + + db.Update(func(btx *bolt.Tx) error { + _, err := btx.CreateBucketIfNotExists(BKTTxs) + if err != nil { + return err + } + _, err = btx.CreateBucketIfNotExists(BKTHeightTxs) + if err != nil { + return err + } + return nil + }) + + return store, nil +} + +func (t *txs) Put(txn *util.Tx) (err error) { + t.Lock() + defer t.Unlock() + + return t.Update(func(tx *bolt.Tx) error { + buf := new(bytes.Buffer) + if err = txn.Serialize(buf); err != nil { + return err + } + + if err = tx.Bucket(BKTTxs).Put(txn.Hash().Bytes(), buf.Bytes()); err != nil { + return err + } + + var key [4]byte + binary.LittleEndian.PutUint32(key[:], txn.Height) + data := tx.Bucket(BKTHeightTxs).Get(key[:]) + + var txMap = make(map[common.Uint256]uint32) + gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) + + txMap[txn.Hash()] = txn.Height + + buf = new(bytes.Buffer) + if err = gob.NewEncoder(buf).Encode(txMap); err != nil { + return err + } + + return tx.Bucket(BKTHeightTxs).Put(key[:], buf.Bytes()) + }) +} + +func (t *txs) Get(hash *common.Uint256) (txn *util.Tx, err error) { + t.RLock() + defer t.RUnlock() + + err = t.View(func(tx *bolt.Tx) error { + data := tx.Bucket(BKTTxs).Get(hash.Bytes()) + txn = new(util.Tx) + return txn.Deserialize(bytes.NewReader(data)) + }) + + return txn, err +} + +func (t *txs) GetIds(height uint32) (txIds []*common.Uint256, err error) { + t.RLock() + defer t.RUnlock() + + err = t.View(func(tx *bolt.Tx) error { + var key [4]byte + binary.LittleEndian.PutUint32(key[:], height) + data := tx.Bucket(BKTHeightTxs).Get(key[:]) + + var txMap = make(map[common.Uint256]uint32) + err = gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) + if err != nil { + return err + } + + txIds = make([]*common.Uint256, 0, len(txMap)) + for hash := range txMap { + var txId common.Uint256 + copy(txId[:], hash[:]) + txIds = append(txIds, &txId) + } + return nil + }) + + return txIds, err +} + +func (t *txs) GetAll() (txs []*util.Tx, err error) { + t.RLock() + defer t.RUnlock() + + err = t.View(func(tx *bolt.Tx) error { + return tx.Bucket(BKTTxs).ForEach(func(k, v []byte) error { + var txn util.Tx + err := txn.Deserialize(bytes.NewReader(v)) + if err != nil { + return err + } + txs = append(txs, &txn) + return nil + }) + }) + + return txs, err +} + +func (t *txs) Del(txId *common.Uint256) (err error) { + t.Lock() + defer t.Unlock() + + return t.DB.Update(func(tx *bolt.Tx) error { + var txn util.Tx + data := tx.Bucket(BKTTxs).Get(txId.Bytes()) + err := txn.Deserialize(bytes.NewReader(data)) + if err != nil { + return err + } + + var key [4]byte + binary.LittleEndian.PutUint32(key[:], txn.Height) + data = tx.Bucket(BKTHeightTxs).Get(key[:]) + + var txMap = make(map[common.Uint256]uint32) + err = gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) + if err != nil { + return err + } + delete(txMap, *txId) + + var buf = new(bytes.Buffer) + if err = gob.NewEncoder(buf).Encode(txMap); err != nil { + return err + } + + return tx.Bucket(BKTHeightTxs).Put(key[:], buf.Bytes()) + }) +} + +func (t *txs) Batch() TxsBatch { + tx, err := t.DB.Begin(true) + if err != nil { + panic(err) + } + + return &txsBatch{Tx: tx} +} + +func (t *txs) Clear() error { + t.Lock() + defer t.Unlock() + + return t.DB.Update(func(tx *bolt.Tx) error { + if err := tx.DeleteBucket(BKTTxs); err != nil { + return err + } + + if err := tx.DeleteBucket(BKTHeightTxs); err != nil { + return err + } + + return nil + }) +} + +func (t *txs) Close() error { + t.Lock() + return nil +} diff --git a/interface/store/txsbatch.go b/interface/store/txsbatch.go new file mode 100644 index 0000000..6427c21 --- /dev/null +++ b/interface/store/txsbatch.go @@ -0,0 +1,165 @@ +package store + +import ( + "bytes" + "encoding/binary" + "encoding/gob" + "github.com/boltdb/bolt" + "sync" + + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +// Ensure txsBatch implement TxsBatch interface. +var _ TxsBatch = (*txsBatch)(nil) + +type txsBatch struct { + sync.Mutex + *bolt.Tx + addTxs []*util.Tx + delTxs []*util.Tx +} + +func (b *txsBatch) Put(tx *util.Tx) error { + b.Lock() + defer b.Unlock() + + buf := new(bytes.Buffer) + if err := tx.Serialize(buf); err != nil { + return err + } + + err := b.Tx.Bucket(BKTTxs).Put(tx.Hash().Bytes(), buf.Bytes()) + if err != nil { + return err + } + + b.addTxs = append(b.addTxs, tx) + return nil +} + +func (b *txsBatch) Del(txId *common.Uint256) error { + b.Lock() + defer b.Unlock() + + var tx util.Tx + data := b.Tx.Bucket(BKTTxs).Get(txId.Bytes()) + err := tx.Deserialize(bytes.NewReader(data)) + if err != nil { + return err + } + + err = b.Tx.Bucket(BKTTxs).Delete(txId.Bytes()) + if err != nil { + return err + } + + b.delTxs = append(b.delTxs, &tx) + return nil +} + +func (b *txsBatch) DelAll(height uint32) error { + b.Lock() + defer b.Unlock() + + var key [4]byte + binary.LittleEndian.PutUint32(key[:], height) + data := b.Tx.Bucket(BKTHeightTxs).Get(key[:]) + + var txMap = make(map[common.Uint256]uint32) + err := gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) + if err != nil { + return err + } + + txBucket := b.Tx.Bucket(BKTTxs) + for txId := range txMap { + if err := txBucket.Delete(txId.Bytes()); err != nil { + return err + } + } + + return b.Tx.Bucket(BKTHeightTxs).Delete(key[:]) +} + +func (b *txsBatch) Commit() error { + b.Lock() + defer b.Unlock() + + index := b.Tx.Bucket(BKTHeightTxs) + + // Put height index for added transactions. + if len(b.addTxs) > 0 { + groups := groupByHeight(b.addTxs) + for height, txs := range groups { + var key [4]byte + binary.LittleEndian.PutUint32(key[:], height) + data := index.Get(key[:]) + + var txMap = make(map[common.Uint256]uint32) + // Ignore decode error, could be first adding. + gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) + + for _, tx := range txs { + txMap[tx.Hash()] = height + } + + var buf = new(bytes.Buffer) + if err := gob.NewEncoder(buf).Encode(txMap); err != nil { + return err + } + + return index.Put(key[:], buf.Bytes()) + } + } + + // Update height index for deleted transactions. + if len(b.delTxs) > 0 { + groups := groupByHeight(b.delTxs) + for height, txs := range groups { + var key [4]byte + binary.LittleEndian.PutUint32(key[:], height) + data := index.Get(key[:]) + + var txMap = make(map[common.Uint256]uint32) + err := gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) + if err != nil { + return err + } + + for _, tx := range txs { + delete(txMap, tx.Hash()) + } + + var buf = new(bytes.Buffer) + if err = gob.NewEncoder(buf).Encode(txMap); err != nil { + return err + } + + return index.Put(key[:], buf.Bytes()) + } + } + + return b.Tx.Commit() +} + +func (b *txsBatch) Rollback() error { + b.Lock() + defer b.Unlock() + return b.Tx.Rollback() +} + +func groupByHeight(txs []*util.Tx) map[uint32][]*util.Tx { + txGroups := make(map[uint32][]*util.Tx) + for _, tx := range txs { + group, ok := txGroups[tx.Height] + if !ok { + txGroups[tx.Height] = append(group, tx) + } else { + group = append(group, tx) + } + } + return txGroups +} From d024d22ba1313fe3cfb4414fa1dfe4fb6d2236f3 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Wed, 12 Sep 2018 11:24:48 +0800 Subject: [PATCH 28/73] initial refactor commit for interface/ package --- interface/db/addrs.go | 91 ---- interface/db/datastore.go | 138 ----- interface/db/headerstore.go | 261 ---------- interface/db/outpoints.go | 81 --- interface/db/proofs.go | 166 ------ interface/db/queue.go | 137 ----- interface/db/queueitem.go | 11 - interface/db/storetx.go | 34 -- interface/db/txs.go | 120 ----- interface/interface.go | 101 ++++ .../{spvservice_test.go => interface_test.go} | 78 +-- interface/keystore.go | 2 +- interface/keystoreimpl.go | 26 +- interface/log.go | 28 + interface/spvservice.go | 487 +++++++++++++++--- interface/spvserviceimpl.go | 352 ------------- 16 files changed, 612 insertions(+), 1501 deletions(-) delete mode 100644 interface/db/addrs.go delete mode 100644 interface/db/datastore.go delete mode 100644 interface/db/headerstore.go delete mode 100644 interface/db/outpoints.go delete mode 100644 interface/db/proofs.go delete mode 100644 interface/db/queue.go delete mode 100644 interface/db/queueitem.go delete mode 100644 interface/db/storetx.go delete mode 100644 interface/db/txs.go create mode 100644 interface/interface.go rename interface/{spvservice_test.go => interface_test.go} (61%) create mode 100644 interface/log.go delete mode 100644 interface/spvserviceimpl.go diff --git a/interface/db/addrs.go b/interface/db/addrs.go deleted file mode 100644 index d5e8e2a..0000000 --- a/interface/db/addrs.go +++ /dev/null @@ -1,91 +0,0 @@ -package db - -import ( - "sync" - - "github.com/elastos/Elastos.ELA.SPV/sdk" - - "github.com/elastos/Elastos.ELA.Utility/common" - - "github.com/boltdb/bolt" -) - -type Addrs interface { - GetFilter() *sdk.AddrFilter - Put(addr *common.Uint168) error - GetAll() []*common.Uint168 -} - -var ( - BKTAddrs = []byte("Addrs") -) - -type AddrStore struct { - *sync.RWMutex - *bolt.DB - filter *sdk.AddrFilter -} - -func NewAddrsDB(db *bolt.DB) (*AddrStore, error) { - store := new(AddrStore) - store.RWMutex = new(sync.RWMutex) - store.DB = db - - db.Update(func(btx *bolt.Tx) error { - _, err := btx.CreateBucketIfNotExists(BKTAddrs) - if err != nil { - return err - } - return nil - }) - - addrs, err := store.getAll() - if err != nil { - return nil, err - } - store.filter = sdk.NewAddrFilter(addrs) - - return store, nil -} - -func (a *AddrStore) GetFilter() *sdk.AddrFilter { - a.Lock() - defer a.Unlock() - return a.filter -} - -func (a *AddrStore) Put(addr *common.Uint168) error { - a.Lock() - defer a.Unlock() - - if a.filter.ContainAddr(*addr) { - return nil - } - - a.filter.AddAddr(addr) - return a.Update(func(tx *bolt.Tx) error { - return tx.Bucket(BKTAddrs).Put(addr.Bytes(), addr.Bytes()) - }) -} - -func (a *AddrStore) GetAll() []*common.Uint168 { - a.RLock() - defer a.RUnlock() - - return a.filter.GetAddrs() -} - -func (a *AddrStore) getAll() (addrs []*common.Uint168, err error) { - err = a.View(func(tx *bolt.Tx) error { - return tx.Bucket(BKTAddrs).ForEach(func(k, v []byte) error { - addr, err := common.Uint168FromBytes(v) - if err != nil { - return err - } - addrs = append(addrs, addr) - return nil - }) - }) - - return addrs, err -} diff --git a/interface/db/datastore.go b/interface/db/datastore.go deleted file mode 100644 index 0024677..0000000 --- a/interface/db/datastore.go +++ /dev/null @@ -1,138 +0,0 @@ -package db - -import ( - "bytes" - "encoding/binary" - "encoding/gob" - "github.com/boltdb/bolt" - "sync" - - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" -) - -type DataStore interface { - Addrs() Addrs - Txs() Txs - Outpoints() Outpoints - Proofs() Proofs - Rollback(height uint32) error - Reset() error - Close() -} - -type DataStoreImpl struct { - *sync.RWMutex - *bolt.DB - addrs *AddrStore - txs *TxStore - outpoints *OutpointStore - proofs *ProofStore -} - -func NewDataStore() (*DataStoreImpl, error) { - db, err := bolt.Open("data_store.bin", 0644, &bolt.Options{InitialMmapSize: 5000000}) - if err != nil { - return nil, err - } - store := new(DataStoreImpl) - store.RWMutex = new(sync.RWMutex) - store.DB = db - - store.addrs, err = NewAddrsDB(db) - if err != nil { - return nil, err - } - - store.txs, err = NewTxsDB(db) - if err != nil { - return nil, err - } - - store.outpoints, err = NewOutpointDB(db) - if err != nil { - return nil, err - } - - store.proofs, err = NewProofsDB(db) - if err != nil { - return nil, err - } - - return store, nil -} - -func (d *DataStoreImpl) Addrs() Addrs { - return d.addrs -} - -func (d *DataStoreImpl) Txs() Txs { - return d.txs -} - -func (d *DataStoreImpl) Outpoints() Outpoints { - return d.outpoints -} - -func (d *DataStoreImpl) Proofs() Proofs { - return d.proofs -} - -func (d *DataStoreImpl) Rollback(height uint32) error { - d.Lock() - defer d.Unlock() - - return d.Update(func(tx *bolt.Tx) error { - var key [4]byte - binary.LittleEndian.PutUint32(key[:], height) - data := tx.Bucket(BKTHeightTxs).Get(key[:]) - - var txMap = make(map[common.Uint256]uint32) - err := gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) - if err != nil { - return err - } - - for hash := range txMap { - var txn StoreTx - data := tx.Bucket(BKTTxs).Get(hash.Bytes()) - if err = txn.Deserialize(bytes.NewReader(data)); err != nil { - return err - } - for index, output := range txn.Outputs { - if d.addrs.GetFilter().ContainAddr(output.ProgramHash) { - outpoint := core.NewOutPoint(txn.Hash(), uint16(index)).Bytes() - tx.Bucket(BKTOps).Delete(outpoint) - } - } - if err = tx.Bucket(BKTTxs).Delete(hash.Bytes()); err != nil { - return err - } - } - return nil - }) -} - -func (d *DataStoreImpl) Reset() error { - d.Lock() - defer d.Unlock() - - return d.Update(func(tx *bolt.Tx) error { - tx.DeleteBucket(BKTAddrs) - tx.DeleteBucket(BKTTxs) - tx.DeleteBucket(BKTHeightTxs) - tx.DeleteBucket(BKTOps) - tx.DeleteBucket(BKTProofs) - return nil - }) -} - -// Close db -func (d *DataStoreImpl) Close() { - d.Lock() - d.addrs.Lock() - d.txs.Lock() - d.outpoints.Lock() - d.proofs.Lock() - d.DB.Close() -} diff --git a/interface/db/headerstore.go b/interface/db/headerstore.go deleted file mode 100644 index 351df28..0000000 --- a/interface/db/headerstore.go +++ /dev/null @@ -1,261 +0,0 @@ -package db - -import ( - "encoding/binary" - "encoding/hex" - "errors" - "fmt" - "math/big" - "sync" - - "github.com/elastos/Elastos.ELA.SPV/store" - "github.com/elastos/Elastos.ELA.Utility/common" - - "github.com/boltdb/bolt" - "github.com/cevaris/ordered_map" -) - -var ( - BKTHeaders = []byte("Headers") - BKTHeightHash = []byte("HeightHash") - BKTChainTip = []byte("ChainTip") - KEYChainTip = []byte("ChainTip") -) - -type HeaderStore struct { - *sync.RWMutex - *bolt.DB - cache *HeaderCache -} - -func NewHeaderStore() (*HeaderStore, error) { - db, err := bolt.Open("headers.bin", 0644, &bolt.Options{InitialMmapSize: 5000000}) - if err != nil { - return nil, err - } - - db.Update(func(btx *bolt.Tx) error { - _, err := btx.CreateBucketIfNotExists(BKTHeaders) - if err != nil { - return err - } - _, err = btx.CreateBucketIfNotExists(BKTHeightHash) - if err != nil { - return err - } - _, err = btx.CreateBucketIfNotExists(BKTChainTip) - if err != nil { - return err - } - return nil - }) - - headers := &HeaderStore{ - RWMutex: new(sync.RWMutex), - DB: db, - cache: newHeaderCache(100), - } - - headers.initCache() - - return headers, nil -} - -func (h *HeaderStore) initCache() { - best, err := h.GetBestHeader() - if err != nil { - return - } - h.cache.tip = best - headers := []*store.StoreHeader{best} - for i := 0; i < 99; i++ { - sh, err := h.GetPrevious(best) - if err != nil { - break - } - headers = append(headers, sh) - } - for i := len(headers) - 1; i >= 0; i-- { - h.cache.Set(headers[i]) - } -} - -func (h *HeaderStore) PutHeader(header *store.StoreHeader, newTip bool) error { - h.Lock() - defer h.Unlock() - - h.cache.Set(header) - if newTip { - h.cache.tip = header - } - return h.Update(func(tx *bolt.Tx) error { - - bytes, err := header.Serialize() - if err != nil { - return err - } - - err = tx.Bucket(BKTHeaders).Put(header.Hash().Bytes(), bytes) - if err != nil { - return err - } - - if newTip { - err = tx.Bucket(BKTChainTip).Put(KEYChainTip, bytes) - if err != nil { - return err - } - } - - var key [4]byte - binary.LittleEndian.PutUint32(key[:], header.Height) - return tx.Bucket(BKTHeightHash).Put(key[:], header.Hash().Bytes()) - }) -} - -func (h *HeaderStore) GetPrevious(header *store.StoreHeader) (*store.StoreHeader, error) { - if header.Height == 1 { - return &store.StoreHeader{TotalWork: new(big.Int)}, nil - } - return h.GetHeader(&header.Previous) -} - -func (h *HeaderStore) GetHeader(hash *common.Uint256) (header *store.StoreHeader, err error) { - h.RLock() - defer h.RUnlock() - - header, err = h.cache.Get(hash) - if err == nil { - return header, nil - } - - err = h.View(func(tx *bolt.Tx) error { - - header, err = getHeader(tx, BKTHeaders, hash.Bytes()) - if err != nil { - return err - } - - return nil - }) - - return header, err -} - -func (h *HeaderStore) GetBestHeader() (header *store.StoreHeader, err error) { - h.RLock() - defer h.RUnlock() - - if h.cache.tip != nil { - return h.cache.tip, nil - } - - err = h.View(func(tx *bolt.Tx) error { - - header, err = getHeader(tx, BKTChainTip, KEYChainTip) - if err != nil { - return err - } - - return nil - }) - - return header, err -} - -func (h *HeaderStore) GetHeaderHash(height uint32) (hash *common.Uint256, err error) { - h.RLock() - defer h.RUnlock() - - err = h.View(func(tx *bolt.Tx) error { - var key [4]byte - binary.LittleEndian.PutUint32(key[:], height) - data := tx.Bucket(BKTHeightHash).Get(key[:]) - hash, err = common.Uint256FromBytes(data) - return err - }) - - if err != nil { - return hash, fmt.Errorf("header hash not exist on height %d", height) - } - - return hash, err -} - -func (h *HeaderStore) Reset() error { - h.Lock() - defer h.Unlock() - - return h.Update(func(tx *bolt.Tx) error { - err := tx.DeleteBucket(BKTHeaders) - if err != nil { - return err - } - - return tx.DeleteBucket(BKTChainTip) - }) -} - -// Close db -func (h *HeaderStore) Close() { - h.Lock() - h.DB.Close() -} - -func getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*store.StoreHeader, error) { - headerBytes := tx.Bucket(bucket).Get(key) - if headerBytes == nil { - return nil, fmt.Errorf("header %s does not exist in database", hex.EncodeToString(key)) - } - - var header store.StoreHeader - err := header.Deserialize(headerBytes) - if err != nil { - return nil, err - } - - return &header, nil -} - -type HeaderCache struct { - sync.RWMutex - size int - tip *store.StoreHeader - headers *ordered_map.OrderedMap -} - -func newHeaderCache(size int) *HeaderCache { - return &HeaderCache{ - size: size, - headers: ordered_map.NewOrderedMap(), - } -} - -func (cache *HeaderCache) pop() { - iter := cache.headers.IterFunc() - k, ok := iter() - if ok { - cache.headers.Delete(k.Key) - } -} - -func (cache *HeaderCache) Set(header *store.StoreHeader) { - cache.Lock() - defer cache.Unlock() - - if cache.headers.Len() > cache.size { - cache.pop() - } - cache.headers.Set(header.Hash().String(), header) -} - -func (cache *HeaderCache) Get(hash *common.Uint256) (*store.StoreHeader, error) { - cache.RLock() - defer cache.RUnlock() - - sh, ok := cache.headers.Get(hash.String()) - if !ok { - return nil, errors.New("Header not found in cache ") - } - return sh.(*store.StoreHeader), nil -} diff --git a/interface/db/outpoints.go b/interface/db/outpoints.go deleted file mode 100644 index 43f6942..0000000 --- a/interface/db/outpoints.go +++ /dev/null @@ -1,81 +0,0 @@ -package db - -import ( - "sync" - - "github.com/elastos/Elastos.ELA/core" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/boltdb/bolt" -) - -var ( - BKTOps = []byte("Ops") -) - -type Outpoints interface { - Put(*core.OutPoint, common.Uint168) error - IsExist(*core.OutPoint) *common.Uint168 - GetAll() ([]*core.OutPoint, error) -} - -type OutpointStore struct { - *sync.RWMutex - *bolt.DB -} - -func NewOutpointDB(db *bolt.DB) (*OutpointStore, error) { - store := new(OutpointStore) - store.RWMutex = new(sync.RWMutex) - store.DB = db - - db.Update(func(btx *bolt.Tx) error { - _, err := btx.CreateBucketIfNotExists(BKTOps) - if err != nil { - return err - } - return nil - }) - - return store, nil -} - -func (t *OutpointStore) Put(op *core.OutPoint, addr common.Uint168) (err error) { - t.Lock() - defer t.Unlock() - return t.Update(func(tx *bolt.Tx) error { - return tx.Bucket(BKTOps).Put(op.Bytes(), addr.Bytes()) - }) -} - -func (t *OutpointStore) IsExist(op *core.OutPoint) (addr *common.Uint168) { - t.RLock() - defer t.RUnlock() - - t.View(func(tx *bolt.Tx) error { - addrBytes := tx.Bucket(BKTOps).Get(op.Bytes()) - var err error - if addr, err = common.Uint168FromBytes(addrBytes); err != nil { - return err - } - return nil - }) - return addr -} - -func (t *OutpointStore) GetAll() (ops []*core.OutPoint, err error) { - t.RLock() - defer t.RUnlock() - - err = t.View(func(tx *bolt.Tx) error { - return tx.Bucket(BKTOps).ForEach(func(k, v []byte) error { - op, err := core.OutPointFromBytes(k) - if err != nil { - return err - } - ops = append(ops, op) - return nil - }) - }) - - return ops, err -} diff --git a/interface/db/proofs.go b/interface/db/proofs.go deleted file mode 100644 index 4b689d1..0000000 --- a/interface/db/proofs.go +++ /dev/null @@ -1,166 +0,0 @@ -package db - -import ( - "bytes" - "encoding/hex" - "errors" - "fmt" - "sync" - - "github.com/boltdb/bolt" - "github.com/elastos/Elastos.ELA/bloom" - "encoding/binary" -) - -type Proofs interface { - // Put a merkle proof of the block - Put(proof *bloom.MerkleProof) error - - // Get a merkle proof of a block - Get(height uint32) (*bloom.MerkleProof, error) - - // Get all merkle proofs in database - GetAll() ([]*bloom.MerkleProof, error) - - // Delete a merkle proof of a block - Delete(height uint32) error -} - -type ProofStore struct { - *sync.RWMutex - *bolt.DB -} - -var ( - BKTProofs = []byte("Proofs") -) - -func NewProofsDB(db *bolt.DB) (*ProofStore, error) { - db.Update(func(btx *bolt.Tx) error { - _, err := btx.CreateBucketIfNotExists(BKTProofs) - if err != nil { - return err - } - return nil - }) - - return &ProofStore{RWMutex: new(sync.RWMutex), DB: db}, nil -} - -// Put a merkle proof of the block -func (db *ProofStore) Put(proof *bloom.MerkleProof) error { - db.Lock() - defer db.Unlock() - - return db.Update(func(tx *bolt.Tx) error { - - bytes, err := serializeProof(proof) - if err != nil { - return err - } - - var key = make([]byte, 4) - binary.LittleEndian.PutUint32(key[:], proof.Height) - - err = tx.Bucket(BKTProofs).Put(key, bytes) - if err != nil { - return err - } - - return nil - }) -} - -// Get a merkle proof of a block -func (db *ProofStore) Get(height uint32) (proof *bloom.MerkleProof, err error) { - db.RLock() - defer db.RUnlock() - - err = db.View(func(tx *bolt.Tx) error { - - var key = make([]byte, 4) - binary.LittleEndian.PutUint32(key[:], height) - - proof, err = getProof(tx, key) - if err != nil { - return err - } - - return nil - }) - - if err != nil { - return nil, err - } - - return proof, err -} - -// Get all merkle proofs in database -func (db *ProofStore) GetAll() (proofs []*bloom.MerkleProof, err error) { - db.RLock() - defer db.RUnlock() - - err = db.View(func(tx *bolt.Tx) error { - - err := tx.Bucket(BKTProofs).ForEach(func(k, v []byte) error { - - proof, err := deserializeProof(v) - if err != nil { - return err - } - - proofs = append(proofs, proof) - - return nil - }) - - if err != nil { - return err - } - - return nil - }) - - return proofs, nil -} - -// Delete a merkle proof of a block -func (db *ProofStore) Delete(height uint32) error { - db.Lock() - defer db.Unlock() - - return db.Update(func(tx *bolt.Tx) error { - var key = make([]byte, 4) - binary.LittleEndian.PutUint32(key[:], height) - - return tx.Bucket(BKTProofs).Delete(key) - }) -} - -func getProof(tx *bolt.Tx, key []byte) (*bloom.MerkleProof, error) { - proofBytes := tx.Bucket(BKTProofs).Get(key) - if proofBytes == nil { - return nil, errors.New(fmt.Sprintf("MerkleProof %s does not exist in database", hex.EncodeToString(key))) - } - - return deserializeProof(proofBytes) -} - -func serializeProof(proof *bloom.MerkleProof) ([]byte, error) { - buf := new(bytes.Buffer) - err := proof.Serialize(buf) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func deserializeProof(body []byte) (*bloom.MerkleProof, error) { - var proof bloom.MerkleProof - err := proof.Deserialize(bytes.NewReader(body)) - if err != nil { - return nil, err - } - return &proof, nil -} diff --git a/interface/db/queue.go b/interface/db/queue.go deleted file mode 100644 index ac9acdf..0000000 --- a/interface/db/queue.go +++ /dev/null @@ -1,137 +0,0 @@ -package db - -import ( - "database/sql" - "fmt" - "sync" - - "github.com/elastos/Elastos.ELA.Utility/common" -) - -type Queue interface { - // Put a queue item to database - Put(item *QueueItem) error - - // Get all items in queue - GetAll() ([]*QueueItem, error) - - // Delete confirmed item in queue - Delete(notifyId, txHash *common.Uint256) error - - // Rollback queue items - Rollback(height uint32) error - - // Reset clear all data in database - Reset() error -} - -const ( - DriverName = "sqlite3" - DBName = "./queue.db" - - CreateQueueDB = `CREATE TABLE IF NOT EXISTS Queue( - NotifyId BLOB NOT NULL, - TxId BLOB NOT NULL, - Height INTEGER NOT NULL); - CREATE INDEX IF NOT EXISTS idx_queue_notify_id ON Queue (NotifyId); - CREATE INDEX IF NOT EXISTS idx_queue_tx_id ON Queue (TxId); - CREATE INDEX IF NOT EXISTS idx_queue_height ON Queue (height);` -) - -type QueueDB struct { - *sync.RWMutex - *sql.DB -} - -func NewQueueDB() (Queue, error) { - db, err := sql.Open(DriverName, DBName) - if err != nil { - fmt.Println("Open sqlite db error:", err) - return nil, err - } - - _, err = db.Exec(CreateQueueDB) - if err != nil { - return nil, err - } - return &QueueDB{RWMutex: new(sync.RWMutex), DB: db}, nil -} - -// Put a queue item to database -func (db *QueueDB) Put(item *QueueItem) error { - db.Lock() - defer db.Unlock() - - sql := "INSERT OR REPLACE INTO Queue(NotifyId, TxId, Height) VALUES(?,?,?)" - _, err := db.Exec(sql, item.NotifyId.Bytes(), item.TxId.Bytes(), item.Height) - if err != nil { - return err - } - - return nil -} - -// Get all items in queue -func (db *QueueDB) GetAll() ([]*QueueItem, error) { - db.RLock() - defer db.RUnlock() - - rows, err := db.Query("SELECT NotifyId, TxId, Height FROM Queue") - if err != nil { - return nil, err - } - - var items []*QueueItem - for rows.Next() { - var notifyIdBytes []byte - var txHashBytes []byte - var height uint32 - err = rows.Scan(¬ifyIdBytes, &txHashBytes, &height) - if err != nil { - return nil, err - } - - notifyId, err := common.Uint256FromBytes(notifyIdBytes) - if err != nil { - return nil, err - } - txHash, err := common.Uint256FromBytes(txHashBytes) - if err != nil { - return nil, err - } - item := &QueueItem{NotifyId: *notifyId, TxId: *txHash, Height: height} - items = append(items, item) - } - - return items, nil -} - -// Delete confirmed item in queue -func (db *QueueDB) Delete(notifyId, txHash *common.Uint256) error { - db.Lock() - defer db.Unlock() - - _, err := db.Exec("DELETE FROM Queue WHERE NotifyId=? AND TxId=?", notifyId.Bytes(), txHash.Bytes()) - if err != nil { - return err - } - - return nil -} - -// Rollback queue items -func (db *QueueDB) Rollback(height uint32) error { - db.Lock() - defer db.Unlock() - - _, err := db.Exec("DELETE FROM Queue WHERE Height=?", height) - return err -} - -func (db *QueueDB) Reset() error { - db.Lock() - defer db.Unlock() - - _, err := db.Exec("DROP TABLE if EXISTS Queue") - return err -} diff --git a/interface/db/queueitem.go b/interface/db/queueitem.go deleted file mode 100644 index 510f7a5..0000000 --- a/interface/db/queueitem.go +++ /dev/null @@ -1,11 +0,0 @@ -package db - -import ( - "github.com/elastos/Elastos.ELA.Utility/common" -) - -type QueueItem struct { - NotifyId common.Uint256 - TxId common.Uint256 - Height uint32 -} diff --git a/interface/db/storetx.go b/interface/db/storetx.go deleted file mode 100644 index f0fecc8..0000000 --- a/interface/db/storetx.go +++ /dev/null @@ -1,34 +0,0 @@ -package db - -import ( - "encoding/binary" - "io" - - "github.com/elastos/Elastos.ELA/core" -) - -type StoreTx struct { - Height uint32 - core.Transaction -} - -func NewStoreTx(tx *core.Transaction, height uint32) *StoreTx { - return &StoreTx{ - Height: height, - Transaction: *tx, - } -} - -func (t *StoreTx) Serialize(buf io.Writer) error { - if err := binary.Write(buf, binary.LittleEndian, t.Height); err != nil { - return err - } - return t.Transaction.Serialize(buf) -} - -func (t *StoreTx) Deserialize(reader io.Reader) error { - if err := binary.Read(reader, binary.LittleEndian, &t.Height); err != nil { - return err - } - return t.Transaction.Deserialize(reader) -} diff --git a/interface/db/txs.go b/interface/db/txs.go deleted file mode 100644 index 124ff71..0000000 --- a/interface/db/txs.go +++ /dev/null @@ -1,120 +0,0 @@ -package db - -import ( - "bytes" - "encoding/binary" - "encoding/gob" - "sync" - - "github.com/elastos/Elastos.ELA.Utility/common" - - "github.com/boltdb/bolt" -) - -type Txs interface { - Put(tx *StoreTx) error - Get(txId *common.Uint256) (*StoreTx, error) - GetIds(height uint32) ([]*common.Uint256, error) -} - -var ( - BKTTxs = []byte("Txs") - BKTHeightTxs = []byte("HeightTxs") -) - -type TxStore struct { - *sync.RWMutex - *bolt.DB -} - -func NewTxsDB(db *bolt.DB) (*TxStore, error) { - store := new(TxStore) - store.RWMutex = new(sync.RWMutex) - store.DB = db - - db.Update(func(btx *bolt.Tx) error { - _, err := btx.CreateBucketIfNotExists(BKTTxs) - if err != nil { - return err - } - _, err = btx.CreateBucketIfNotExists(BKTHeightTxs) - if err != nil { - return err - } - return nil - }) - - return store, nil -} - -func (t *TxStore) Put(txn *StoreTx) (err error) { - t.Lock() - defer t.Unlock() - - return t.Update(func(tx *bolt.Tx) error { - buf := new(bytes.Buffer) - if err = txn.Serialize(buf); err != nil { - return err - } - - if err = tx.Bucket(BKTTxs).Put(txn.Hash().Bytes(), buf.Bytes()); err != nil { - return err - } - - var key [4]byte - binary.LittleEndian.PutUint32(key[:], txn.Height) - data := tx.Bucket(BKTHeightTxs).Get(key[:]) - - var txMap = make(map[common.Uint256]uint32) - gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) - - txMap[txn.Hash()] = txn.Height - - buf = new(bytes.Buffer) - if err = gob.NewEncoder(buf).Encode(txMap); err != nil { - return err - } - - return tx.Bucket(BKTHeightTxs).Put(key[:], buf.Bytes()) - }) -} - -func (t *TxStore) Get(hash *common.Uint256) (txn *StoreTx, err error) { - t.RLock() - defer t.RUnlock() - - err = t.View(func(tx *bolt.Tx) error { - data := tx.Bucket(BKTTxs).Get(hash.Bytes()) - txn = new(StoreTx) - return txn.Deserialize(bytes.NewReader(data)) - }) - - return txn, err -} - -func (t *TxStore) GetIds(height uint32) (txIds []*common.Uint256, err error) { - t.RLock() - defer t.RUnlock() - - err = t.View(func(tx *bolt.Tx) error { - var key [4]byte - binary.LittleEndian.PutUint32(key[:], height) - data := tx.Bucket(BKTHeightTxs).Get(key[:]) - - var txMap = make(map[common.Uint256]uint32) - err = gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) - if err != nil { - return err - } - - txIds = make([]*common.Uint256, 0, len(txMap)) - for hash := range txMap { - var txId common.Uint256 - copy(txId[:], hash[:]) - txIds = append(txIds, &txId) - } - return nil - }) - - return txIds, err -} diff --git a/interface/interface.go b/interface/interface.go new file mode 100644 index 0000000..62be03a --- /dev/null +++ b/interface/interface.go @@ -0,0 +1,101 @@ +package _interface + +import ( + "github.com/elastos/Elastos.ELA.SPV/database" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/bloom" + "github.com/elastos/Elastos.ELA/core" +) + +// SPV service config +type Config struct { + // The magic number that specify which network to connect. + Magic uint32 + + // The foundation address of the genesis block, which is different between + // MainNet, TestNet, RegNet etc. + Foundation string + + // The public seed peers addresses. + SeedList []string + + // DefaultPort is the default port for public peers provide services. + DefaultPort uint16 + + // The minimum target outbound connections. + MinOutbound int + + // The maximum connected peers. + MaxConnections int + + // Rollback callbacks that, the transactions + // on the given height has been rollback + OnRollback func(height uint32) +} + +/* +SPV service is the interface to interactive with the SPV (Simplified Payment Verification) +service implementation running background, you can register specific accounts that you are +interested in and receive transaction notifications of these accounts. +*/ +type SPVService interface { + // RegisterTransactionListener register the listener to receive transaction notifications + // listeners must be registered before call Start() method, or some notifications will go missing. + RegisterTransactionListener(TransactionListener) error + + // After receive the transaction callback, call this method + // to confirm that the transaction with the given ID was handled, + // so the transaction will be removed from the notify queue. + // the notifyId is the key to specify which listener received this notify. + SubmitTransactionReceipt(notifyId common.Uint256, txId common.Uint256) error + + // To verify if a transaction is valid + // This method is useful when receive a transaction from other peer + VerifyTransaction(bloom.MerkleProof, core.Transaction) error + + // Send a transaction to the P2P network + SendTransaction(core.Transaction) error + + // Get headers database + HeaderStore() database.Headers + + // Start the SPV service + Start() error + + // ClearData delete all data stores data including HeaderStore and DataStore. + ClearData() error +} + +const ( + // FlagNotifyConfirmed indicates if this transaction should be callback after reach the confirmed height, + // by default 6 confirmations are needed according to the protocol + FlagNotifyConfirmed = 1 << 0 + + // FlagNotifyInSyncing indicates if notify this listener when SPV is in syncing. + FlagNotifyInSyncing = 1 << 1 +) + +/* +Register this listener into the IService RegisterTransactionListener() method +to receive transaction notifications. +*/ +type TransactionListener interface { + // The address this listener interested + Address() string + + // Type() indicates which transaction type this listener are interested + Type() core.TransactionType + + // Flags control the notification actions by the given flag + Flags() uint64 + + // Notify() is the method to callback the received transaction + // with the merkle tree proof to verify it, the notifyId is key of this + // notify message and it must be submitted with the receipt together. + Notify(notifyId common.Uint256, proof bloom.MerkleProof, tx core.Transaction) +} + +func NewSPVService(config *Config) (SPVService, error) { + return newService(config) +} diff --git a/interface/spvservice_test.go b/interface/interface_test.go similarity index 61% rename from interface/spvservice_test.go rename to interface/interface_test.go index 059f543..d934112 100644 --- a/interface/spvservice_test.go +++ b/interface/interface_test.go @@ -1,25 +1,29 @@ package _interface import ( - "bytes" - "crypto/rand" - "encoding/binary" "fmt" "testing" - "github.com/elastos/Elastos.ELA.SPV/log" + "github.com/elastos/Elastos.ELA.SPV/blockchain" + spvpeer "github.com/elastos/Elastos.ELA.SPV/peer" + "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" + "github.com/elastos/Elastos.ELA.SPV/sync" "github.com/elastos/Elastos.ELA.Utility/common" - . "github.com/elastos/Elastos.ELA/bloom" - . "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA.Utility/elalog" + "github.com/elastos/Elastos.ELA.Utility/p2p/addrmgr" + "github.com/elastos/Elastos.ELA.Utility/p2p/connmgr" + "github.com/elastos/Elastos.ELA.Utility/p2p/server" + "github.com/elastos/Elastos.ELA/bloom" + "github.com/elastos/Elastos.ELA/core" ) -var spv SPVService +var service SPVService type TxListener struct { address string - txType TransactionType + txType core.TransactionType flags uint64 } @@ -27,7 +31,7 @@ func (l *TxListener) Address() string { return l.address } -func (l *TxListener) Type() TransactionType { +func (l *TxListener) Type() core.TransactionType { return l.txType } @@ -35,14 +39,14 @@ func (l *TxListener) Flags() uint64 { return l.flags } -func (l *TxListener) Notify(id common.Uint256, proof MerkleProof, tx Transaction) { +func (l *TxListener) Notify(id common.Uint256, proof bloom.MerkleProof, tx core.Transaction) { fmt.Printf("Receive notify ID: %s, Type: %s\n", id.String(), tx.TxType.Name()) - err := spv.VerifyTransaction(proof, tx) + err := service.VerifyTransaction(proof, tx) if err != nil { fmt.Println("Verify transaction error:", err) } // Submit transaction receipt - spv.SubmitTransactionReceipt(id, tx.Hash()) + service.SubmitTransactionReceipt(id, tx.Hash()) } func (l *TxListener) Rollback(height uint32) {} @@ -51,14 +55,14 @@ func TestGetListenerKey(t *testing.T) { var key1, key2 common.Uint256 listener := &TxListener{ address: "ENTogr92671PKrMmtWo3RLiYXfBTXUe13Z", - txType: CoinBase, + txType: core.CoinBase, flags: FlagNotifyConfirmed | FlagNotifyInSyncing, } key1 = getListenerKey(listener) key2 = getListenerKey(&TxListener{ address: "ENTogr92671PKrMmtWo3RLiYXfBTXUe13Z", - txType: CoinBase, + txType: core.CoinBase, flags: FlagNotifyConfirmed | FlagNotifyInSyncing, }) if !key1.IsEqual(key2) { @@ -77,7 +81,7 @@ func TestGetListenerKey(t *testing.T) { // same address, flags different type key1 = getListenerKey(listener) - listener.txType = TransferAsset + listener.txType = core.TransferAsset key2 = getListenerKey(listener) if key1.IsEqual(key2) { t.Errorf("listeners with different type got same key %s", key1.String()) @@ -96,45 +100,49 @@ func TestGetListenerKey(t *testing.T) { } func TestNewSPVService(t *testing.T) { - log.Init(0, 5, 20) - - var id = make([]byte, 8) - var clientId uint64 - var err error - rand.Read(id) - binary.Read(bytes.NewReader(id), binary.LittleEndian, clientId) - - config := SPVServiceConfig{ - ClientId:clientId, - Magic: config.Values().Magic, - Foundation:config.Values().Foundation, - Seeds: config.Values().SeedList, - MinOutbound: 8, + addrmgr.UseLogger(elalog.Stdout) + connmgr.UseLogger(elalog.Stdout) + sdk.UseLogger(elalog.Stdout) + //rpc.UseLogger(logger) + //peer.UseLogger(elalog.Stdout) + spvpeer.UseLogger(elalog.Stdout) + server.UseLogger(elalog.Stdout) + blockchain.UseLogger(elalog.Stdout) + sync.UseLogger(elalog.Stdout) + UseLogger(elalog.Stdout) + + config := &Config{ + Magic: config.Values().Magic, + Foundation: config.Values().Foundation, + SeedList: config.Values().SeedList, + DefaultPort: config.Values().DefaultPort, + MinOutbound: 8, MaxConnections: 100, } - spv, err = NewSPVService(config) + + service, err := NewSPVService(config) if err != nil { t.Error("NewSPVService error %s", err.Error()) } confirmedListener := &TxListener{ address: "ENTogr92671PKrMmtWo3RLiYXfBTXUe13Z", - txType: CoinBase, + txType: core.CoinBase, flags: FlagNotifyConfirmed | FlagNotifyInSyncing, } unconfirmedListener := &TxListener{ address: "Ef2bDPwcUKguteJutJQCmjX2wgHVfkJ2Wq", - txType: TransferAsset, + txType: core.TransferAsset, flags: 0, } // Set on transaction confirmed callback - spv.RegisterTransactionListener(confirmedListener) - spv.RegisterTransactionListener(unconfirmedListener) + service.RegisterTransactionListener(confirmedListener) + service.RegisterTransactionListener(unconfirmedListener) // Start spv service - err = spv.Start() + err = service.Start() if err != nil { t.Error("Start SPV service error: ", err) } diff --git a/interface/keystore.go b/interface/keystore.go index f281a4f..3360187 100644 --- a/interface/keystore.go +++ b/interface/keystore.go @@ -39,5 +39,5 @@ type Account interface { } func NewKeystore() Keystore { - return &KeystoreImpl{} + return &keystore{} } diff --git a/interface/keystoreimpl.go b/interface/keystoreimpl.go index 086228a..77ed67b 100644 --- a/interface/keystoreimpl.go +++ b/interface/keystoreimpl.go @@ -1,24 +1,24 @@ package _interface import ( - "github.com/elastos/Elastos.ELA.SPV/spvwallet" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/client" ) -type KeystoreImpl struct { - keystore spvwallet.Keystore +type keystore struct { + keystore client.Keystore } // This method will open or create a keystore with the given password -func (impl *KeystoreImpl) Open(password string) (Keystore, error) { +func (impl *keystore) Open(password string) (Keystore, error) { var err error // Try to open keystore first - impl.keystore, err = spvwallet.OpenKeystore([]byte(password)) + impl.keystore, err = client.OpenKeystore([]byte(password)) if err == nil { return impl, nil } // Try to create a keystore - impl.keystore, err = spvwallet.CreateKeystore([]byte(password)) + impl.keystore, err = client.CreateKeystore([]byte(password)) if err != nil { return nil, err } @@ -26,19 +26,19 @@ func (impl *KeystoreImpl) Open(password string) (Keystore, error) { return impl, nil } -func (impl *KeystoreImpl) ChangePassword(old, new string) error { +func (impl *keystore) ChangePassword(old, new string) error { return impl.keystore.ChangePassword([]byte(old), []byte(new)) } -func (impl *KeystoreImpl) MainAccount() Account { +func (impl *keystore) MainAccount() Account { return impl.keystore.MainAccount() } -func (impl *KeystoreImpl) NewAccount() Account { +func (impl *keystore) NewAccount() Account { return impl.keystore.NewAccount() } -func (impl *KeystoreImpl) GetAccounts() []Account { +func (impl *keystore) GetAccounts() []Account { var accounts []Account for _, account := range impl.keystore.GetAccounts() { accounts = append(accounts, account) @@ -46,11 +46,11 @@ func (impl *KeystoreImpl) GetAccounts() []Account { return accounts } -func (impl *KeystoreImpl) Json() (string, error) { +func (impl *keystore) Json() (string, error) { return impl.keystore.Json() } -func (impl *KeystoreImpl) FromJson(str string, password string) error { - impl.keystore = new(spvwallet.KeystoreImpl) +func (impl *keystore) FromJson(str string, password string) error { + impl.keystore = new(client.KeystoreImpl) return impl.keystore.FromJson(str, password) } diff --git a/interface/log.go b/interface/log.go new file mode 100644 index 0000000..de15bcd --- /dev/null +++ b/interface/log.go @@ -0,0 +1,28 @@ +package _interface + +import ( + "github.com/elastos/Elastos.ELA.Utility/elalog" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log elalog.Logger + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = elalog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using elalog. +func UseLogger(logger elalog.Logger) { + log = logger +} diff --git a/interface/spvservice.go b/interface/spvservice.go index ee60bd5..7d54a82 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -1,89 +1,454 @@ package _interface import ( - "github.com/elastos/Elastos.ELA.SPV/store" + "bytes" + "crypto/sha256" + "errors" + "fmt" + "os" + "os/signal" + + "github.com/elastos/Elastos.ELA.SPV/database" + "github.com/elastos/Elastos.ELA.SPV/interface/store" + "github.com/elastos/Elastos.ELA.SPV/sdk" + "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA/bloom" "github.com/elastos/Elastos.ELA/core" ) -// SPV service config -type SPVServiceConfig struct { - Magic uint32 - Foundation string - ClientId uint64 - Seeds []string - MinOutbound int - MaxConnections int +const minPeersForSync = 3 + +type spvservice struct { + sdk.IService + headers store.HeaderStore + db store.DataStore + rollback func(height uint32) + listeners map[common.Uint256]TransactionListener } -/* -SPV service is the interface to interactive with the SPV (Simplified Payment Verification) -service implementation running background, you can register specific accounts that you are -interested in and receive transaction notifications of these accounts. -*/ -type SPVService interface { - // RegisterTransactionListener register the listener to receive transaction notifications - // listeners must be registered before call Start() method, or some notifications will go missing. - RegisterTransactionListener(TransactionListener) error +func newService(cfg *Config) (*spvservice, error) { + headerStore, err := store.NewHeaderStore() + if err != nil { + return nil, err + } - // After receive the transaction callback, call this method - // to confirm that the transaction with the given ID was handled, - // so the transaction will be removed from the notify queue. - // the notifyId is the key to specify which listener received this notify. - SubmitTransactionReceipt(notifyId common.Uint256, txId common.Uint256) error + dataStore, err := store.NewDataStore() + if err != nil { + return nil, err + } - // To verify if a transaction is valid - // This method is useful when receive a transaction from other peer - VerifyTransaction(bloom.MerkleProof, core.Transaction) error + service := &spvservice{ + headers: headerStore, + db: dataStore, + rollback: cfg.OnRollback, + listeners: make(map[common.Uint256]TransactionListener), + } - // Send a transaction to the P2P network - SendTransaction(core.Transaction) error + chainStore := database.NewDefaultChainDB(headerStore, service) - // Get headers database - HeaderStore() store.HeaderStore + serviceCfg := &sdk.Config{ + Magic: cfg.Magic, + SeedList: cfg.SeedList, + DefaultPort: cfg.DefaultPort, + MaxPeers: cfg.MaxConnections, + MinPeersForSync: minPeersForSync, + Foundation: cfg.Foundation, + ChainStore: chainStore, + GetFilterData: service.GetFilterData, + StateNotifier: service, + } - // Start the SPV service - Start() error + service.IService, err = sdk.NewService(serviceCfg) + if err != nil { + return nil, err + } - // ResetStores clear all data stores data including HeaderStore, ProofStore, AddrsStore, TxsStore etc. - ResetStores() error + return service, nil } -const ( - // FlagNotifyConfirmed indicates if this transaction should be callback after reach the confirmed height, - // by default 6 confirmations are needed according to the protocol - FlagNotifyConfirmed = 1 << 0 +func (s *spvservice) RegisterTransactionListener(listener TransactionListener) error { + address, err := common.Uint168FromAddress(listener.Address()) + if err != nil { + return fmt.Errorf("address %s is not a valied address", listener.Address()) + } + key := getListenerKey(listener) + if _, ok := s.listeners[key]; ok { + return fmt.Errorf("listener with address: %s type: %s flags: %d already registered", + listener.Address(), listener.Type().Name(), listener.Flags()) + } + s.listeners[key] = listener + return s.db.Addrs().Put(address) +} - // FlagNotifyInSyncing indicates if notify this listener when SPV is in syncing. - FlagNotifyInSyncing = 1 << 1 -) +func (s *spvservice) SubmitTransactionReceipt(notifyId, txHash common.Uint256) error { + return s.db.Que().Del(¬ifyId, &txHash) +} + +func (s *spvservice) VerifyTransaction(proof bloom.MerkleProof, tx core.Transaction) error { + // Get Header from main chain + header, err := s.headers.Get(&proof.BlockHash) + if err != nil { + return errors.New("can not get block from main chain") + } + + // Check if merkleroot is match + merkleBlock := msg.MerkleBlock{ + Header: &header.Header, + Transactions: proof.Transactions, + Hashes: proof.Hashes, + Flags: proof.Flags, + } + txIds, err := bloom.CheckMerkleBlock(merkleBlock) + if err != nil { + return fmt.Errorf("check merkle branch failed, %s", err.Error()) + } + if len(txIds) == 0 { + return fmt.Errorf("invalid transaction proof, no transactions found") + } + + // Check if transaction hash is match + match := false + for _, txId := range txIds { + if *txId == tx.Hash() { + match = true + break + } + } + if !match { + return fmt.Errorf("transaction hash not match proof") + } + + return nil +} + +func (s *spvservice) SendTransaction(tx core.Transaction) error { + return s.IService.SendTransaction(tx) +} + +func (s *spvservice) HeaderStore() database.Headers { + return s.headers +} + +func (s *spvservice) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { + ops, err := s.db.Ops().GetAll() + if err != nil { + log.Error("[SPV_SERVICE] GetData error ", err) + } + + return s.db.Addrs().GetAll(), ops +} + +// Batch returns a TxBatch instance for transactions batch +// commit, this can get better performance when commit a bunch +// of transactions within a block. +func (s *spvservice) Batch() database.TxBatch { + return &txBatch{ + db: s.db, + batch: s.db.Batch(), + rollback: s.rollback, + } +} + +// CommitTx save a transaction into database, and return +// if it is a false positive and error. +func (s *spvservice) CommitTx(tx *util.Tx) (bool, error) { + hits := make(map[common.Uint168]struct{}) + for index, output := range tx.Outputs { + if s.db.Addrs().GetFilter().ContainAddr(output.ProgramHash) { + outpoint := core.NewOutPoint(tx.Hash(), uint16(index)) + if err := s.db.Ops().Put(outpoint, output.ProgramHash); err != nil { + return false, err + } + hits[output.ProgramHash] = struct{}{} + } + } + + for _, input := range tx.Inputs { + if addr := s.db.Ops().IsExist(&input.Previous); addr != nil { + hits[*addr] = struct{}{} + } + } + + if len(hits) == 0 { + return true, nil + } + + for _, listener := range s.listeners { + hash, _ := common.Uint168FromAddress(listener.Address()) + if _, ok := hits[*hash]; ok { + s.queueMessageByListener(listener, &tx.Transaction, tx.Height) + } + } + + return false, s.db.Txs().Put(util.NewTx(tx.Transaction, tx.Height)) +} + +// HaveTx returns if the transaction already saved in database +// by it's id. +func (s *spvservice) HaveTx(txId *common.Uint256) (bool, error) { + tx, err := s.db.Txs().Get(txId) + return tx != nil, err +} + +// GetTxs returns all transactions within the given height. +func (s *spvservice) GetTxs(height uint32) ([]*util.Tx, error) { + return nil, nil +} + +// RemoveTxs delete all transactions on the given height. Return +// how many transactions are deleted from database. +func (s *spvservice) RemoveTxs(height uint32) (int, error) { + batch := s.db.Batch() + if err := batch.DelAll(height); err != nil { + return 0, batch.Rollback() + } + return 0, batch.Commit() +} + +// TransactionAccepted will be invoked after a transaction sent by +// SendTransaction() method has been accepted. Notice: this method needs at +// lest two connected peers to work. +func (s *spvservice) TransactionAccepted(tx *util.Tx) {} + +// TransactionRejected will be invoked if a transaction sent by SendTransaction() +// method has been rejected. +func (s *spvservice) TransactionRejected(tx *util.Tx) {} + +// TransactionConfirmed will be invoked after a transaction sent by +// SendTransaction() method has been packed into a block. +func (s *spvservice) TransactionConfirmed(tx *util.Tx) {} + +// BlockCommitted will be invoked when a block and transactions within it are +// successfully committed into database. +func (s *spvservice) BlockCommitted(block *util.Block) { + // Look up for queued transactions + items, err := s.db.Que().GetAll() + if err != nil { + return + } + for _, item := range items { + // Get header + header, err := s.headers.GetByHeight(item.Height) + if err != nil { + log.Errorf("query merkle proof at height %d failed, %s", item.Height, err.Error()) + continue + } + + // Get transaction from db + storeTx, err := s.db.Txs().Get(&item.TxId) + if err != nil { + log.Errorf("query transaction failed, txId %s", item.TxId.String()) + continue + } + + // Notify listeners + s.notifyTransaction( + item.NotifyId, + bloom.MerkleProof{ + BlockHash: header.Hash(), + Height: header.Height, + Transactions: block.NumTxs, + Hashes: block.Hashes, + Flags: block.Flags, + }, + storeTx.Transaction, + header.Height-item.Height, + ) + } +} -/* -Register this listener into the SPVService RegisterTransactionListener() method -to receive transaction notifications. -*/ -type TransactionListener interface { - // The address this listener interested - Address() string +func (s *spvservice) Start() error { + // Handle interrupt signal + quit := make(chan struct{}) + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt) + go func() { + for range signals { + log.Trace("SPV service shutting down...") + s.IService.Stop() + quit <- struct{}{} + } + }() - // Type() indicates which transaction type this listener are interested - Type() core.TransactionType + // Start SPV service + s.IService.Start() - // Flags control the notification actions by the given flag - Flags() uint64 + <-quit - // Notify() is the method to callback the received transaction - // with the merkle tree proof to verify it, the notifyId is key of this - // notify message and it must be submitted with the receipt together. - Notify(notifyId common.Uint256, proof bloom.MerkleProof, tx core.Transaction) + return nil +} + +func (s *spvservice) ClearData() error { + if err := s.headers.Clear(); err != nil { + log.Warnf("Clear header store error %s", err.Error()) + } + if err := s.db.Clear(); err != nil { + log.Warnf("Clear data store error %s", err.Error()) + } + return nil +} + +func (s *spvservice) Clear() error { + return s.db.Clear() +} + +func (s *spvservice) Close() error { + return s.db.Close() +} + +func (s *spvservice) queueMessageByListener( + listener TransactionListener, tx *core.Transaction, height uint32) { + // skip unpacked transaction + if height == 0 { + return + } + + // skip transactions that not match the require type + if listener.Type() != tx.TxType { + return + } + + // queue message + s.db.Que().Put(&store.QueItem{ + NotifyId: getListenerKey(listener), + TxId: tx.Hash(), + Height: height, + }) +} + +func (s *spvservice) notifyTransaction( + notifyId common.Uint256, proof bloom.MerkleProof, tx core.Transaction, confirmations uint32) { + + listener, ok := s.listeners[notifyId] + if !ok { + return + } + + // Get transaction id + txId := tx.Hash() + + // Remove notifications if FlagNotifyInSyncing not set + if s.IService.IsCurrent() == false && + listener.Flags()&FlagNotifyInSyncing != FlagNotifyInSyncing { + + if listener.Flags()&FlagNotifyConfirmed == FlagNotifyConfirmed { + if confirmations >= getConfirmations(tx) { + s.db.Que().Del(¬ifyId, &txId) + } + } else { + s.db.Que().Del(¬ifyId, &txId) + } + return + } + + // Notify listener + if listener.Flags()&FlagNotifyConfirmed == FlagNotifyConfirmed { + if confirmations >= getConfirmations(tx) { + go listener.Notify(notifyId, proof, tx) + } + } else { + go listener.Notify(notifyId, proof, tx) + } +} + +func getListenerKey(listener TransactionListener) common.Uint256 { + buf := new(bytes.Buffer) + addr, _ := common.Uint168FromAddress(listener.Address()) + common.WriteElements(buf, addr[:], listener.Type(), listener.Flags()) + return sha256.Sum256(buf.Bytes()) +} + +func getConfirmations(tx core.Transaction) uint32 { + // TODO user can set confirmations attribute in transaction, + // if the confirmation attribute is set, use it instead of default value + if tx.TxType == core.CoinBase { + return 100 + } + return DefaultConfirmations +} + +type txBatch struct { + db store.DataStore + batch store.DataBatch + heights []uint32 + rollback func(height uint32) +} - // Rollback callbacks that, the transactions - // on the given height has been rollback - Rollback(height uint32) +// AddTx add a store transaction operation into batch, and return +// if it is a false positive and error. +func (b *txBatch) AddTx(tx *util.Tx) (bool, error) { + hits := 0 + ops := make(map[*core.OutPoint]common.Uint168) + for index, output := range tx.Outputs { + if b.db.Addrs().GetFilter().ContainAddr(output.ProgramHash) { + outpoint := core.NewOutPoint(tx.Hash(), uint16(index)) + ops[outpoint] = output.ProgramHash + hits++ + } + } + + for _, input := range tx.Inputs { + if addr := b.db.Ops().IsExist(&input.Previous); addr != nil { + hits++ + } + } + + if hits == 0 { + return true, nil + } + + for op, addr := range ops { + if err := b.batch.Ops().Put(op, addr); err != nil { + return false, err + } + } + + return false, b.batch.Txs().Put(tx) } -func NewSPVService(config SPVServiceConfig) (SPVService, error) { - return NewSPVServiceImpl(config) +// DelTx add a delete transaction operation into batch. +func (b *txBatch) DelTx(txId *common.Uint256) error { + tx, err := b.db.Txs().Get(txId) + if err != nil { + return err + } + + for index := range tx.Outputs { + outpoint := core.NewOutPoint(tx.Hash(), uint16(index)) + b.batch.Ops().Del(outpoint) + } + + return b.batch.Txs().Del(txId) +} + +// DelTxs add a delete transactions on given height operation. +func (b *txBatch) DelTxs(height uint32) error { + if b.rollback != nil { + b.heights = append(b.heights, height) + } + return b.batch.DelAll(height) +} + +// Rollback cancel all operations in current batch. +func (b *txBatch) Rollback() error { + return b.batch.Rollback() +} + +// Commit the added transactions into database. +func (b *txBatch) Commit() error { + err := b.batch.Commit() + if err != nil { + return err + } + + go func(heights []uint32) { + for _, height := range heights { + b.rollback(height) + } + }(b.heights) + + return nil } diff --git a/interface/spvserviceimpl.go b/interface/spvserviceimpl.go deleted file mode 100644 index 3b3ad8a..0000000 --- a/interface/spvserviceimpl.go +++ /dev/null @@ -1,352 +0,0 @@ -package _interface - -import ( - "bytes" - "crypto/sha256" - "errors" - "fmt" - "os" - "os/signal" - - "github.com/elastos/Elastos.ELA.SPV/interface/db" - "github.com/elastos/Elastos.ELA.SPV/log" - "github.com/elastos/Elastos.ELA.SPV/sdk" - "github.com/elastos/Elastos.ELA.SPV/store" - - "github.com/elastos/Elastos.ELA.SPV/net" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA/bloom" - "github.com/elastos/Elastos.ELA/core" -) - -type SPVServiceImpl struct { - sdk.SPVService - headers *db.HeaderStore - dataStore db.DataStore - queue db.Queue - listeners map[common.Uint256]TransactionListener -} - -func NewSPVServiceImpl(config SPVServiceConfig) (*SPVServiceImpl, error) { - var err error - service := new(SPVServiceImpl) - service.headers, err = db.NewHeaderStore() - if err != nil { - return nil, err - } - - service.dataStore, err = db.NewDataStore() - if err != nil { - return nil, err - } - - service.queue, err = db.NewQueueDB() - if err != nil { - return nil, err - } - - serverPeerConfig := net.ServerPeerConfig{ - Magic: config.Magic, - Version: p2p.EIP001Version, - PeerId: config.ClientId, - Port: 0, - Seeds: config.Seeds, - MinOutbound: config.MinOutbound, - MaxConnections: config.MaxConnections, - } - - serviceConfig := sdk.SPVServiceConfig{ - Server: net.NewServerPeer(serverPeerConfig), - Foundation: config.Foundation, - HeaderStore: service.headers, - GetFilterData: service.GetFilterData, - CommitTx: service.CommitTx, - OnBlockCommitted: service.OnBlockCommitted, - OnRollback: service.OnRollback, - } - - service.SPVService, err = sdk.GetSPVService(serviceConfig) - if err != nil { - return nil, err - } - - service.listeners = make(map[common.Uint256]TransactionListener) - - return service, nil -} - -func (service *SPVServiceImpl) RegisterTransactionListener(listener TransactionListener) error { - address, err := common.Uint168FromAddress(listener.Address()) - if err != nil { - return fmt.Errorf("address %s is not a valied address", listener.Address()) - } - key := getListenerKey(listener) - if _, ok := service.listeners[key]; ok { - return fmt.Errorf("listener with address: %s type: %s flags: %d already registered", - listener.Address(), listener.Type().Name(), listener.Flags()) - } - service.listeners[key] = listener - return service.dataStore.Addrs().Put(address) -} - -func (service *SPVServiceImpl) SubmitTransactionReceipt(notifyId, txHash common.Uint256) error { - return service.queue.Delete(¬ifyId, &txHash) -} - -func (service *SPVServiceImpl) VerifyTransaction(proof bloom.MerkleProof, tx core.Transaction) error { - // Get Header from main chain - header, err := service.headers.GetHeader(&proof.BlockHash) - if err != nil { - return errors.New("can not get block from main chain") - } - - // Check if merkleroot is match - merkleBlock := msg.MerkleBlock{ - Header: &header.Header, - Transactions: proof.Transactions, - Hashes: proof.Hashes, - Flags: proof.Flags, - } - txIds, err := bloom.CheckMerkleBlock(merkleBlock) - if err != nil { - return fmt.Errorf("check merkle branch failed, %s", err.Error()) - } - if len(txIds) == 0 { - return fmt.Errorf("invalid transaction proof, no transactions found") - } - - // Check if transaction hash is match - match := false - for _, txId := range txIds { - if *txId == tx.Hash() { - match = true - break - } - } - if !match { - return fmt.Errorf("transaction hash not match proof") - } - - return nil -} - -func (service *SPVServiceImpl) SendTransaction(tx core.Transaction) error { - _, err := service.SPVService.SendTransaction(tx) - return err -} - -func (service *SPVServiceImpl) HeaderStore() store.HeaderStore { - return service.headers -} - -func (service *SPVServiceImpl) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { - ops, err := service.dataStore.Outpoints().GetAll() - if err != nil { - log.Error("[SPV_SERVICE] GetData error ", err) - } - - return service.dataStore.Addrs().GetAll(), ops -} - -func (service *SPVServiceImpl) CommitTx(tx *core.Transaction, height uint32) (bool, error) { - hits := make(map[common.Uint168]struct{}) - for index, output := range tx.Outputs { - if service.dataStore.Addrs().GetFilter().ContainAddr(output.ProgramHash) { - outpoint := core.NewOutPoint(tx.Hash(), uint16(index)) - if err := service.dataStore.Outpoints().Put(outpoint, output.ProgramHash); err != nil { - return false, err - } - hits[output.ProgramHash] = struct{}{} - } - } - - for _, input := range tx.Inputs { - if addr := service.dataStore.Outpoints().IsExist(&input.Previous); addr != nil { - hits[*addr] = struct{}{} - } - } - - if len(hits) == 0 { - return true, nil - } - - for _, listener := range service.listeners { - hash, _ := common.Uint168FromAddress(listener.Address()) - if _, ok := hits[*hash]; ok { - service.queueMessageByListener(listener, tx, height) - } - } - - return false, service.dataStore.Txs().Put(db.NewStoreTx(tx, height)) -} - -func (service *SPVServiceImpl) OnBlockCommitted(block *msg.MerkleBlock, txs []*core.Transaction) { - header := block.Header.(*core.Header) - - // Store merkle proof - proof := bloom.MerkleProof{ - BlockHash: header.Hash(), - Height: header.Height, - Transactions: block.Transactions, - Hashes: block.Hashes, - Flags: block.Flags, - } - - if err := service.dataStore.Proofs().Put(&proof); err != nil { - log.Errorf("[SPV_SERVICE] store merkle proof failed, error %s", err.Error()) - return - } - - // Look up for queued transactions - items, err := service.queue.GetAll() - if err != nil { - return - } - for _, item := range items { - // Get proof from db - proof, err := service.dataStore.Proofs().Get(item.Height) - if err != nil { - log.Errorf("query merkle proof at height %d failed, %s", item.Height, err.Error()) - continue - } - // Get transaction from db - storeTx, err := service.dataStore.Txs().Get(&item.TxId) - if err != nil { - log.Errorf("query transaction failed, txId %s", item.TxId.String()) - continue - } - - // Notify listeners - service.notifyTransaction(item.NotifyId, *proof, storeTx.Transaction, header.Height-item.Height) - } -} - -// Overwrite OnRollback() method in SPVWallet -func (service *SPVServiceImpl) OnRollback(height uint32) error { - err := service.dataStore.Rollback(height) - if err != nil { - log.Warnf("Rollback data store error %s", err.Error()) - } - err = service.queue.Rollback(height) - if err != nil { - log.Warnf("Rollback transaction notify queue error %s", err.Error()) - } - service.notifyRollback(height) - return nil -} - -func (service *SPVServiceImpl) Start() error { - // Handle interrupt signal - stop := make(chan int, 1) - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt) - go func() { - for range signals { - log.Trace("SPV service shutting down...") - service.Stop() - stop <- 1 - } - }() - - // Start SPV service - service.SPVService.Start() - - <-stop - - return nil -} - -func (service *SPVServiceImpl) ResetStores() error { - err := service.headers.Reset() - if err != nil { - log.Warnf("Reset header store error %s", err.Error()) - } - err = service.dataStore.Reset() - if err != nil { - log.Warnf("Reset data store error %s", err.Error()) - } - err = service.queue.Reset() - if err != nil { - log.Warnf("Reset transaction notify queue store error %s", err.Error()) - } - return nil -} - -func (service *SPVServiceImpl) queueMessageByListener( - listener TransactionListener, tx *core.Transaction, height uint32) { - // skip unpacked transaction - if height == 0 { - return - } - - // skip transactions that not match the require type - if listener.Type() != tx.TxType { - return - } - - // queue message - service.queue.Put(&db.QueueItem{ - NotifyId: getListenerKey(listener), - TxId: tx.Hash(), - Height: height, - }) -} - -func (service *SPVServiceImpl) notifyTransaction( - notifyId common.Uint256, proof bloom.MerkleProof, tx core.Transaction, confirmations uint32) { - - listener, ok := service.listeners[notifyId] - if !ok { - return - } - - // Get transaction id - txId := tx.Hash() - - // Remove notifications if FlagNotifyInSyncing not set - if service.SPVService.ChainState() == sdk.SYNCING && - listener.Flags()&FlagNotifyInSyncing != FlagNotifyInSyncing { - - if listener.Flags()&FlagNotifyConfirmed == FlagNotifyConfirmed { - if confirmations >= getConfirmations(tx) { - service.queue.Delete(¬ifyId, &txId) - } - } else { - service.queue.Delete(¬ifyId, &txId) - } - return - } - - // Notify listener - if listener.Flags()&FlagNotifyConfirmed == FlagNotifyConfirmed { - if confirmations >= getConfirmations(tx) { - go listener.Notify(notifyId, proof, tx) - } - } else { - go listener.Notify(notifyId, proof, tx) - } -} - -func (service *SPVServiceImpl) notifyRollback(height uint32) { - for _, listener := range service.listeners { - go listener.Rollback(height) - } -} - -func getListenerKey(listener TransactionListener) common.Uint256 { - buf := new(bytes.Buffer) - addr, _ := common.Uint168FromAddress(listener.Address()) - common.WriteElements(buf, addr[:], listener.Type(), listener.Flags()) - return sha256.Sum256(buf.Bytes()) -} - -func getConfirmations(tx core.Transaction) uint32 { - // TODO user can set confirmations attribute in transaction, - // if the confirmation attribute is set, use it instead of default value - if tx.TxType == core.CoinBase { - return 100 - } - return DefaultConfirmations -} From bcaf8b07dec04f1f5ba8353c3b2a2c9ac5698078 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Wed, 12 Sep 2018 18:51:10 +0800 Subject: [PATCH 29/73] prevent sending duplicate filterload message to same peer --- sync/manager.go | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/sync/manager.go b/sync/manager.go index 952b88c..9f254ac 100644 --- a/sync/manager.go +++ b/sync/manager.go @@ -104,6 +104,7 @@ type peerSyncState struct { requestedBlocks map[common.Uint256]struct{} receivedBlocks uint32 badBlocks uint32 + filterUpdating bool receivedTxs uint32 falsePositives uint32 } @@ -244,25 +245,35 @@ func (sm *SyncManager) getSyncCandidates() []*peer.Peer { // updateBloomFilter update the bloom filter and send it to the given peer. func (sm *SyncManager) updateBloomFilter(p *peer.Peer) { - msg := sm.cfg.UpdateFilter().GetFilterLoadMsg() - log.Debugf("Update bloom filter %v, %d, %d", msg.Filter, msg.Tweak, msg.HashFuncs) - doneChan := make(chan struct{}) - p.QueueMessage(msg, doneChan) + state, ok := sm.peerStates[p] + if !ok { + log.Warnf("Update bloom filter for unknown peer %s", p) + return + } + + if state.filterUpdating { + return + } + state.filterUpdating = true + + go func(state *peerSyncState) { + + msg := sm.cfg.UpdateFilter().GetFilterLoadMsg() + log.Debugf("Update bloom filter %v, %d, %d", msg.Filter, msg.Tweak, msg.HashFuncs) + done := make(chan struct{}) + p.QueueMessage(msg, done) - go func(p *peer.Peer) { select { - case <-doneChan: + case <-done: // Reset false positive state. - state, ok := sm.peerStates[p] - if ok { - state.receivedTxs = 0 - state.falsePositives = 0 - } + state.filterUpdating = false + state.receivedTxs = 0 + state.falsePositives = 0 case <-p.Quit(): - return + break } - }(p) + }(state) } // handleNewPeerMsg deals with new peers that have signalled they may @@ -487,8 +498,6 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { func (sm *SyncManager) haveInventory(invVect *msg.InvVect) bool { switch invVect.Type { case msg.InvTypeBlock: - fallthrough - case msg.InvTypeFilteredBlock: // Ask chain if the block is known to it in any form (main // chain, side chain, or orphan). return sm.cfg.Chain.HaveBlock(&invVect.Hash) From 3fc6f56296279829a7a2f6299602919819d33b51 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 13 Sep 2018 11:42:21 +0800 Subject: [PATCH 30/73] fix chain HaveBlock() method always return false issue --- blockchain/chain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blockchain/chain.go b/blockchain/chain.go index 24d1386..1614dc2 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -209,8 +209,8 @@ func (b *BlockChain) getCommonAncestor(bestHeader, prevTip *util.Header) (*util. // // This function is safe for concurrent access. func (b *BlockChain) HaveBlock(hash *common.Uint256) bool { - header, err := b.db.Headers().Get(hash) - return err != nil && header != nil + header, _ := b.db.Headers().Get(hash) + return header != nil } // LatestBlockLocator returns a block locator for current last block, From e500713734fc00735021df7f14ab2e5d4641e613 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 13 Sep 2018 18:47:22 +0800 Subject: [PATCH 31/73] implement fprate package to handle false positives --- fprate/fprate.go | 35 ++++++++++++++++++++ fprate/fprate_test.go | 23 +++++++++++++ peer/peer.go | 10 ++++++ sdk/bloom.go | 4 ++- sdk/spvservice.go | 17 ++++------ sync/manager.go | 77 +++++++++++++------------------------------ 6 files changed, 100 insertions(+), 66 deletions(-) create mode 100644 fprate/fprate.go create mode 100644 fprate/fprate_test.go diff --git a/fprate/fprate.go b/fprate/fprate.go new file mode 100644 index 0000000..ce7039b --- /dev/null +++ b/fprate/fprate.go @@ -0,0 +1,35 @@ +package fprate + +import "github.com/elastos/Elastos.ELA.SPV/util" + +const ( + DefaultFalsePositiveRate float64 = 0.0005 + ReducedFalsePositiveRate float64 = 0.00005 +) + +type FpRate struct { + averageTxPerBlock float64 + fpRate float64 +} + +func (r *FpRate) Update(block *util.Block, fps uint32) (fpRate float64) { + // moving average number of tx-per-block + r.averageTxPerBlock = r.averageTxPerBlock*0.999 + float64(block.NumTxs)*0.001 + + // 1% low pass filter, also weights each block by total transactions, compared to the avarage + r.fpRate = r.fpRate*(1.0-0.01*float64(block.NumTxs)/r.averageTxPerBlock) + + 0.01*float64(fps)/r.averageTxPerBlock + + return r.fpRate +} + +func (r *FpRate) Reset() { + r.fpRate = ReducedFalsePositiveRate +} + +func NewFpRate() *FpRate { + return &FpRate{ + averageTxPerBlock: 1400, + fpRate: ReducedFalsePositiveRate, + } +} diff --git a/fprate/fprate_test.go b/fprate/fprate_test.go new file mode 100644 index 0000000..1c20b98 --- /dev/null +++ b/fprate/fprate_test.go @@ -0,0 +1,23 @@ +package fprate + +import ( + "math/rand" + "testing" + + "github.com/elastos/Elastos.ELA.SPV/util" +) + +func TestFpRate_Update(t *testing.T) { + + fpRate := NewFpRate() + + block := &util.Block{ + Header: util.Header{ + NumTxs: 1, + }, + } + for i := 0; i < 5000; i++ { + fp := uint32(rand.Intn(3)) + t.Logf("FpRate %f on height %d, fp %d", fpRate.Update(block, fp), i+1, fp) + } +} diff --git a/peer/peer.go b/peer/peer.go index 9679ecb..4b267e2 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -113,6 +113,11 @@ func (p *Peer) sendMessage(out outMsg) { } func (p *Peer) handleMessage(peer *peer.Peer, message p2p.Message) { + // Return if peer already disconnected. + if !p.Connected() { + return + } + switch m := message.(type) { case *msg.Inv: p.stallControl <- message @@ -247,6 +252,11 @@ func (p *Peer) blockHandler() { // NotifyOnBlock message and clear cached data. notifyBlock := func() { + // Return if peer already disconnected. + if !p.Connected() { + return + } + // Notify OnBlock. p.cfg.OnBlock(p, &util.Block{ Header: *header, diff --git a/sdk/bloom.go b/sdk/bloom.go index 5a018cc..75fcc36 100644 --- a/sdk/bloom.go +++ b/sdk/bloom.go @@ -1,6 +1,8 @@ package sdk import ( + "github.com/elastos/Elastos.ELA.SPV/fprate" + "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/bloom" "github.com/elastos/Elastos.ELA/core" @@ -9,7 +11,7 @@ import ( // Create a new bloom filter instance // elements are how many elements will be added to this filter. func NewBloomFilter(elements uint32) *bloom.Filter { - return bloom.NewFilter(elements, 0, 0.00003) + return bloom.NewFilter(elements, 0, fprate.ReducedFalsePositiveRate) } // Build a bloom filter by giving the interested addresses and outpoints diff --git a/sdk/spvservice.go b/sdk/spvservice.go index 198ecad..197f81f 100644 --- a/sdk/spvservice.go +++ b/sdk/spvservice.go @@ -412,18 +412,13 @@ func (s *service) onBlock(sp *spvpeer.Peer, block *util.Block) { done := make(chan struct{}) s.syncManager.QueueBlock(block, sp, done) - // Use a new goroutine to prevent blocking. - go func() { - select { - case <-done: - s.txQueue <- &blockMsg{block: block} - if s.cfg.StateNotifier != nil { - s.cfg.StateNotifier.BlockCommitted(block) - } - case <-sp.Quit(): - return + select { + case <-done: + s.txQueue <- &blockMsg{block: block} + if s.cfg.StateNotifier != nil { + s.cfg.StateNotifier.BlockCommitted(block) } - }() + } } func (s *service) onTx(sp *spvpeer.Peer, tx *core.Transaction) { diff --git a/sync/manager.go b/sync/manager.go index 9f254ac..b9831f9 100644 --- a/sync/manager.go +++ b/sync/manager.go @@ -4,6 +4,7 @@ import ( "sync/atomic" "github.com/elastos/Elastos.ELA.SPV/blockchain" + "github.com/elastos/Elastos.ELA.SPV/fprate" "github.com/elastos/Elastos.ELA.SPV/peer" "github.com/elastos/Elastos.ELA.SPV/util" @@ -25,10 +26,6 @@ const ( // maxBadBlockRate is the maximum bad blocks rate of received blocks. maxBadBlockRate float64 = 0.001 - // maxFalsePositiveRate is the maximum false positive rate of received - // transactions. - maxFalsePositiveRate float64 = 0.0001 - // maxRequestedBlocks is the maximum number of requested block // hashes to store in memory. maxRequestedBlocks = msg.MaxInvPerMsg @@ -104,19 +101,13 @@ type peerSyncState struct { requestedBlocks map[common.Uint256]struct{} receivedBlocks uint32 badBlocks uint32 - filterUpdating bool - receivedTxs uint32 - falsePositives uint32 + fpRate *fprate.FpRate } func (s *peerSyncState) badBlockRate() float64 { return float64(s.badBlocks) / float64(s.receivedBlocks) } -func (s *peerSyncState) falsePosRate() float64 { - return float64(s.falsePositives) / float64(s.receivedTxs) -} - // SyncManager is used to communicate block related messages with peers. The // SyncManager is started as by executing Start() in a goroutine. Once started, // it selects peers to sync from and starts the initial block download. Once the @@ -208,6 +199,9 @@ func (sm *SyncManager) startSync() { } func (sm *SyncManager) syncWith(p *peer.Peer) { + // Update bloom filter first, before start requesting blocks. + sm.updateBloomFilter(p) + // Clear the requestedBlocks if the sync peer changes, otherwise we // may ignore blocks we need that the last sync peer failed to send. sm.requestedBlocks = make(map[common.Uint256]struct{}) @@ -245,35 +239,9 @@ func (sm *SyncManager) getSyncCandidates() []*peer.Peer { // updateBloomFilter update the bloom filter and send it to the given peer. func (sm *SyncManager) updateBloomFilter(p *peer.Peer) { - state, ok := sm.peerStates[p] - if !ok { - log.Warnf("Update bloom filter for unknown peer %s", p) - return - } - - if state.filterUpdating { - return - } - state.filterUpdating = true - - go func(state *peerSyncState) { - - msg := sm.cfg.UpdateFilter().GetFilterLoadMsg() - log.Debugf("Update bloom filter %v, %d, %d", msg.Filter, msg.Tweak, msg.HashFuncs) - done := make(chan struct{}) - p.QueueMessage(msg, done) - - select { - case <-done: - // Reset false positive state. - state.filterUpdating = false - state.receivedTxs = 0 - state.falsePositives = 0 - - case <-p.Quit(): - break - } - }(state) + msg := sm.cfg.UpdateFilter().GetFilterLoadMsg() + log.Debugf("Update bloom filter %v, %d, %d", msg.Filter, msg.Tweak, msg.HashFuncs) + p.QueueMessage(msg, nil) } // handleNewPeerMsg deals with new peers that have signalled they may @@ -293,10 +261,9 @@ func (sm *SyncManager) handleNewPeerMsg(peer *peer.Peer) { syncCandidate: isSyncCandidate, requestedTxns: make(map[common.Uint256]struct{}), requestedBlocks: make(map[common.Uint256]struct{}), + fpRate: fprate.NewFpRate(), } - sm.updateBloomFilter(peer) - // Start syncing by choosing the best candidate if needed. if isSyncCandidate && sm.syncPeer == nil { sm.startSync() @@ -372,10 +339,6 @@ func (sm *SyncManager) handleTxMsg(tmsg *txMsg) { if fp { log.Debugf("Tx %s from Peer%d is a false positive.", txHash.String(), peer.ID()) - state.falsePositives++ - if state.falsePosRate() > maxFalsePositiveRate { - sm.updateBloomFilter(peer) - } } } @@ -398,8 +361,7 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { // If we didn't ask for this block then the peer is misbehaving. block := bmsg.block - header := block.Header - blockHash := header.Hash() + blockHash := block.Hash() if _, exists = state.requestedBlocks[blockHash]; !exists { peer.Disconnect() return @@ -448,18 +410,25 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { return } - // Check false positive rate. - state.falsePositives += fps - if state.falsePosRate() > maxFalsePositiveRate { - sm.updateBloomFilter(peer) - } - // We can exit here if the block is already known if !newBlock { log.Debugf("Received duplicate block %s", blockHash.String()) return } + // Check false positive rate. + fpRate := state.fpRate.Update(block, fps) + if fpRate > fprate.DefaultFalsePositiveRate*10 { + log.Warnf("bloom filter false positive rate %f too high," + + " disconnecting...", fpRate) + peer.Disconnect() + return + } + if newHeight+500 < peer.Height() && fpRate > fprate.ReducedFalsePositiveRate*10 { + sm.updateBloomFilter(peer) + state.fpRate.Reset() + } + log.Infof("Received block %s at height %d", blockHash.String(), newHeight) // Check reorg From b3f50ea751488791ed8e253cbf7d46dfdc6cffe0 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 15 Sep 2018 16:03:46 +0800 Subject: [PATCH 32/73] remove CommitTx method from txstore interface, add TransactionAnnounce method into StateNotifier to instead --- blockchain/chain.go | 12 ++-- database/chainstore.go | 8 +-- database/defaultdb.go | 12 +--- database/headersonly.go | 10 +--- database/txsdb.go | 8 +-- interface/spvservice.go | 79 +++++++++++--------------- peer/peer.go | 9 +-- sdk/interface.go | 7 ++- sdk/spvservice.go | 20 +++---- spvwallet/store/sqlite/txs.go | 1 - spvwallet/wallet.go | 101 ++++++++++++---------------------- sync/config.go | 5 +- sync/manager.go | 11 +--- util/block.go | 4 +- util/header.go | 2 +- 15 files changed, 106 insertions(+), 183 deletions(-) diff --git a/blockchain/chain.go b/blockchain/chain.go index 1614dc2..cc38193 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -2,12 +2,12 @@ package blockchain import ( "errors" - "github.com/elastos/Elastos.ELA.SPV/database" - "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA/core" "math/big" "sync" + "github.com/elastos/Elastos.ELA.SPV/database" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" ) @@ -53,10 +53,6 @@ func New(foundation string, db database.ChainStore) (*BlockChain, error) { return chain, nil } -func (b *BlockChain) CommitTx(tx *core.Transaction) (bool, error) { - return b.db.StoreTx(&util.Tx{Transaction: *tx, Height: 0}) -} - func (b *BlockChain) CommitBlock(block *util.Block) (newTip, reorg bool, newHeight, fps uint32, err error) { b.lock.Lock() defer b.lock.Unlock() @@ -109,7 +105,7 @@ func (b *BlockChain) CommitBlock(block *util.Block) (newTip, reorg bool, newHeig newHeight = parentHeader.Height + 1 header.Height = newHeight header.TotalWork = cumulativeWork - fps, err = b.db.StoreBlock(block, newTip) + fps, err = b.db.CommitBlock(block, newTip) if err != nil { return newTip, reorg, 0, 0, err } diff --git a/database/chainstore.go b/database/chainstore.go index c26abc8..6190272 100644 --- a/database/chainstore.go +++ b/database/chainstore.go @@ -12,13 +12,9 @@ type ChainStore interface { // all blockchain headers. Headers() Headers - // StoreBlock save a block into database, returns how many + // CommitBlock save a block into database, returns how many // false positive transactions are and error. - StoreBlock(block *util.Block, newTip bool) (fps uint32, err error) - - // StoreTx save a transaction into database, and return - // if it is a false positive and error. - StoreTx(tx *util.Tx) (bool, error) + CommitBlock(block *util.Block, newTip bool) (fps uint32, err error) // Rollback delete all transactions after the reorg point, // it is used when blockchain reorganized. diff --git a/database/defaultdb.go b/database/defaultdb.go index ccf8e6e..c937a88 100644 --- a/database/defaultdb.go +++ b/database/defaultdb.go @@ -13,9 +13,9 @@ func (d *defaultChainDB) Headers() Headers { return d.h } -// StoreBlock save a block into database, returns how many +// CommitBlock save a block into database, returns how many // false positive transactions are and error. -func (d *defaultChainDB) StoreBlock(block *util.Block, newTip bool) (fps uint32, err error) { +func (d *defaultChainDB) CommitBlock(block *util.Block, newTip bool) (fps uint32, err error) { err = d.h.Put(&block.Header, newTip) if err != nil { return 0, err @@ -28,7 +28,7 @@ func (d *defaultChainDB) StoreBlock(block *util.Block, newTip bool) (fps uint32, batch := d.t.Batch() for _, tx := range block.Transactions { - fp, err := batch.AddTx(tx) + fp, err := batch.PutTx(util.NewTx(*tx, block.Height)) if err != nil { return 0, batch.Rollback() } @@ -40,12 +40,6 @@ func (d *defaultChainDB) StoreBlock(block *util.Block, newTip bool) (fps uint32, return fps, batch.Commit() } -// StoreTx save a transaction into database, and return -// if it is a false positive and error. -func (d *defaultChainDB) StoreTx(tx *util.Tx) (bool, error) { - return d.t.CommitTx(tx) -} - // RollbackTo delete all transactions after the reorg point, // it is used when blockchain reorganized. func (d *defaultChainDB) Rollback(reorg *util.Header) error { diff --git a/database/headersonly.go b/database/headersonly.go index bacc34a..511c026 100644 --- a/database/headersonly.go +++ b/database/headersonly.go @@ -14,18 +14,12 @@ func (h *headersOnlyChainDB) Headers() Headers { return h.db } -// StoreBlock save a block into database, returns how many +// CommitBlock save a block into database, returns how many // false positive transactions are and error. -func (h *headersOnlyChainDB) StoreBlock(block *util.Block, newTip bool) (fps uint32, err error) { +func (h *headersOnlyChainDB) CommitBlock(block *util.Block, newTip bool) (fps uint32, err error) { return fps, h.db.Put(&block.Header, newTip) } -// StoreTx save a transaction into database, and return -// if it is a false positive and error. -func (h *headersOnlyChainDB) StoreTx(tx *util.Tx) (bool, error) { - return false, nil -} - // RollbackTo delete all transactions after the reorg point, // it is used when blockchain reorganized. func (h *headersOnlyChainDB) Rollback(reorg *util.Header) error { diff --git a/database/txsdb.go b/database/txsdb.go index 21f79ad..6663acc 100644 --- a/database/txsdb.go +++ b/database/txsdb.go @@ -14,10 +14,6 @@ type TxsDB interface { // of transactions within a block. Batch() TxBatch - // CommitTx save a transaction into database, and return - // if it is a false positive and error. - CommitTx(tx *util.Tx) (bool, error) - // HaveTx returns if the transaction already saved in database // by it's id. HaveTx(txId *common.Uint256) (bool, error) @@ -31,9 +27,9 @@ type TxsDB interface { } type TxBatch interface { - // AddTx add a store transaction operation into batch, and return + // PutTx add a store transaction operation into batch, and return // if it is a false positive and error. - AddTx(tx *util.Tx) (bool, error) + PutTx(tx *util.Tx) (bool, error) // DelTx add a delete transaction operation into batch. DelTx(txId *common.Uint256) error diff --git a/interface/spvservice.go b/interface/spvservice.go index 7d54a82..67cc84d 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -152,40 +152,6 @@ func (s *spvservice) Batch() database.TxBatch { } } -// CommitTx save a transaction into database, and return -// if it is a false positive and error. -func (s *spvservice) CommitTx(tx *util.Tx) (bool, error) { - hits := make(map[common.Uint168]struct{}) - for index, output := range tx.Outputs { - if s.db.Addrs().GetFilter().ContainAddr(output.ProgramHash) { - outpoint := core.NewOutPoint(tx.Hash(), uint16(index)) - if err := s.db.Ops().Put(outpoint, output.ProgramHash); err != nil { - return false, err - } - hits[output.ProgramHash] = struct{}{} - } - } - - for _, input := range tx.Inputs { - if addr := s.db.Ops().IsExist(&input.Previous); addr != nil { - hits[*addr] = struct{}{} - } - } - - if len(hits) == 0 { - return true, nil - } - - for _, listener := range s.listeners { - hash, _ := common.Uint168FromAddress(listener.Address()) - if _, ok := hits[*hash]; ok { - s.queueMessageByListener(listener, &tx.Transaction, tx.Height) - } - } - - return false, s.db.Txs().Put(util.NewTx(tx.Transaction, tx.Height)) -} - // HaveTx returns if the transaction already saved in database // by it's id. func (s *spvservice) HaveTx(txId *common.Uint256) (bool, error) { @@ -208,14 +174,17 @@ func (s *spvservice) RemoveTxs(height uint32) (int, error) { return 0, batch.Commit() } +// TransactionAnnounce will be invoked when received a new announced transaction. +func (s *spvservice) TransactionAnnounce(tx *core.Transaction) {} + // TransactionAccepted will be invoked after a transaction sent by // SendTransaction() method has been accepted. Notice: this method needs at // lest two connected peers to work. -func (s *spvservice) TransactionAccepted(tx *util.Tx) {} +func (s *spvservice) TransactionAccepted(tx *core.Transaction) {} // TransactionRejected will be invoked if a transaction sent by SendTransaction() // method has been rejected. -func (s *spvservice) TransactionRejected(tx *util.Tx) {} +func (s *spvservice) TransactionRejected(tx *core.Transaction) {} // TransactionConfirmed will be invoked after a transaction sent by // SendTransaction() method has been packed into a block. @@ -371,32 +340,33 @@ func getConfirmations(tx core.Transaction) uint32 { } type txBatch struct { - db store.DataStore - batch store.DataBatch - heights []uint32 - rollback func(height uint32) + db store.DataStore + batch store.DataBatch + heights []uint32 + rollback func(height uint32) + listeners map[common.Uint256]TransactionListener } -// AddTx add a store transaction operation into batch, and return +// PutTx add a store transaction operation into batch, and return // if it is a false positive and error. -func (b *txBatch) AddTx(tx *util.Tx) (bool, error) { - hits := 0 +func (b *txBatch) PutTx(tx *util.Tx) (bool, error) { + hits := make(map[common.Uint168]struct{}) ops := make(map[*core.OutPoint]common.Uint168) for index, output := range tx.Outputs { if b.db.Addrs().GetFilter().ContainAddr(output.ProgramHash) { outpoint := core.NewOutPoint(tx.Hash(), uint16(index)) ops[outpoint] = output.ProgramHash - hits++ + hits[output.ProgramHash] = struct{}{} } } for _, input := range tx.Inputs { if addr := b.db.Ops().IsExist(&input.Previous); addr != nil { - hits++ + hits[*addr] = struct{}{} } } - if hits == 0 { + if len(hits) == 0 { return true, nil } @@ -406,6 +376,23 @@ func (b *txBatch) AddTx(tx *util.Tx) (bool, error) { } } + for _, listener := range b.listeners { + hash, _ := common.Uint168FromAddress(listener.Address()) + if _, ok := hits[*hash]; ok { + // skip transactions that not match the require type + if listener.Type() != tx.TxType { + continue + } + + // queue message + b.batch.Que().Put(&store.QueItem{ + NotifyId: getListenerKey(listener), + TxId: tx.Hash(), + Height: tx.Height, + }) + } + } + return false, b.batch.Txs().Put(tx) } diff --git a/peer/peer.go b/peer/peer.go index 4b267e2..4aa88c4 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -248,7 +248,7 @@ func (p *Peer) blockHandler() { // Data caches for the downloading block. var header *util.Header var pendingTxs map[common.Uint256]struct{} - var txs []*util.Tx + var txs []*core.Transaction // NotifyOnBlock message and clear cached data. notifyBlock := func() { @@ -314,7 +314,7 @@ out: } // Initiate transactions cache. - txs = make([]*util.Tx, 0, len(pendingTxs)) + txs = make([]*core.Transaction, 0, len(pendingTxs)) case *msg.Tx: // Not in block downloading mode, just notify new transaction. @@ -335,10 +335,7 @@ out: } // Save downloaded transaction to cache. - txs = append(txs, &util.Tx{ - Transaction: *tx, - Height: header.Height, - }) + txs = append(txs, tx) // Remove transaction from pending list. delete(pendingTxs, txId) diff --git a/sdk/interface.go b/sdk/interface.go index a6a3683..a3ead57 100644 --- a/sdk/interface.go +++ b/sdk/interface.go @@ -33,14 +33,17 @@ type IService interface { // StateNotifier exposes methods to notify status changes of transactions and blocks. type StateNotifier interface { + // TransactionAnnounce will be invoked when received a new announced transaction. + TransactionAnnounce(tx *core.Transaction) + // TransactionAccepted will be invoked after a transaction sent by // SendTransaction() method has been accepted. Notice: this method needs at // lest two connected peers to work. - TransactionAccepted(tx *util.Tx) + TransactionAccepted(tx *core.Transaction) // TransactionRejected will be invoked if a transaction sent by SendTransaction() // method has been rejected. - TransactionRejected(tx *util.Tx) + TransactionRejected(tx *core.Transaction) // TransactionConfirmed will be invoked after a transaction sent by // SendTransaction() method has been packed into a block. diff --git a/sdk/spvservice.go b/sdk/spvservice.go index 197f81f..1adf0f7 100644 --- a/sdk/spvservice.go +++ b/sdk/spvservice.go @@ -85,6 +85,9 @@ func NewSPVService(cfg *Config) (*service, error) { syncCfg := sync.NewDefaultConfig(chain, service.updateFilter) syncCfg.MaxPeers = maxPeers syncCfg.MinPeersForSync = minPeersForSync + if cfg.StateNotifier != nil { + syncCfg.TransactionAnnounce = cfg.StateNotifier.TransactionAnnounce + } syncManager, err := sync.New(syncCfg) if err != nil { return nil, err @@ -274,11 +277,7 @@ out: // Use a new goroutine do the invoke to prevent blocking. go func(tx *core.Transaction) { if s.cfg.StateNotifier != nil { - s.cfg.StateNotifier.TransactionAccepted( - &util.Tx{ - Transaction: *tx, - Height: 0, - }) + s.cfg.StateNotifier.TransactionAccepted(tx) } }(txMsg.tx) } @@ -303,11 +302,7 @@ out: // Use a new goroutine do the invoke to prevent blocking. go func(tx *core.Transaction) { if s.cfg.StateNotifier != nil { - s.cfg.StateNotifier.TransactionRejected( - &util.Tx{ - Transaction: *tx, - Height: 0, - }) + s.cfg.StateNotifier.TransactionRejected(tx) } }(txMsg.tx) } @@ -315,10 +310,9 @@ out: case *blockMsg: // Loop through all packed transactions, see if match to any // sent transactions. - confirmedTxs := make(map[common.Uint256]*util.Tx) + confirmedTxs := make(map[common.Uint256]*core.Transaction) for _, tx := range tmsg.block.Transactions { txId := tx.Hash() - tx.Height = tmsg.block.Height if _, ok := unconfirmed[txId]; ok { confirmedTxs[txId] = tx @@ -345,7 +339,7 @@ out: if s.cfg.StateNotifier != nil { s.cfg.StateNotifier.TransactionConfirmed(tx) } - }(tx) + }(util.NewTx(*tx, tmsg.block.Height)) } } case <-retryTicker.C: diff --git a/spvwallet/store/sqlite/txs.go b/spvwallet/store/sqlite/txs.go index 04f275e..24c0181 100644 --- a/spvwallet/store/sqlite/txs.go +++ b/spvwallet/store/sqlite/txs.go @@ -81,7 +81,6 @@ func (t *txs) GetAll() ([]*sutil.Tx, error) { // Fetch all unconfirmed transactions from database func (t *txs) GetAllUnconfirmed() ([]*sutil.Tx, error) { - // TODO implement memcache to get better performance. return t.GetAllFrom(0) } diff --git a/spvwallet/wallet.go b/spvwallet/wallet.go index b3626bd..aa49129 100644 --- a/spvwallet/wallet.go +++ b/spvwallet/wallet.go @@ -54,53 +54,6 @@ func (w *Wallet) Batch() database.TxBatch { } } -// CommitTx save a transaction into database, and return -// if it is a false positive and error. -func (w *Wallet) CommitTx(tx *util.Tx) (bool, error) { - // In this SPV implementation, CommitTx only invoked on new transactions - // that are unconfirmed, we just check if this transaction is a false - // positive and store it into database. NOTICE: at this moment, we didn't - // change UTXOs and STXOs. - txId := tx.Hash() - - // We already have this transaction. - ok := w.txIds.Get(txId) - if ok { - return false, nil - } - - hits := 0 - // Check if any UTXOs of this wallet are used. - for _, input := range tx.Inputs { - stxo, _ := w.db.UTXOs().Get(&input.Previous) - if stxo != nil { - hits++ - } - } - - // Check if there are any output to this wallet address. - for _, output := range tx.Outputs { - if w.getAddrFilter().ContainAddr(output.ProgramHash) { - hits++ - } - } - - // If no hits, no need to save transaction - if hits == 0 { - return true, nil - } - - // Save transaction as unconfirmed. - err := w.db.Txs().Put(sutil.NewTx(tx.Transaction, 0)) - if err != nil { - return false, err - } - - w.txIds.Add(txId) - - return false, nil -} - // HaveTx returns if the transaction already saved in database // by it's id. func (w *Wallet) HaveTx(txId *common.Uint256) (bool, error) { @@ -177,16 +130,28 @@ func (w *Wallet) loadAddrFilter() *sdk.AddrFilter { return w.filter } +// TransactionAnnounce will be invoked when received a new announced transaction. +func (w *Wallet) TransactionAnnounce(tx *core.Transaction) { + // TODO + // Save transaction as unconfirmed. + err := w.db.Txs().Put(sutil.NewTx(*tx, 0)) + if err != nil { + return + } + + w.txIds.Add(tx.Hash()) +} + // TransactionAccepted will be invoked after a transaction sent by // SendTransaction() method has been accepted. Notice: this method needs at // lest two connected peers to work. -func (w *Wallet) TransactionAccepted(tx *util.Tx) { +func (w *Wallet) TransactionAccepted(tx *core.Transaction) { // TODO } // TransactionRejected will be invoked if a transaction sent by SendTransaction() // method has been rejected. -func (w *Wallet) TransactionRejected(tx *util.Tx) { +func (w *Wallet) TransactionRejected(tx *core.Transaction) { // TODO } @@ -201,21 +166,23 @@ func (w *Wallet) TransactionConfirmed(tx *util.Tx) { // BlockCommitted will be invoked when a block and transactions within it are // successfully committed into database. func (w *Wallet) BlockCommitted(block *util.Block) { - if w.IsCurrent() { - w.db.State().PutHeight(block.Height) - // Get all unconfirmed transactions - txs, err := w.db.Txs().GetAllUnconfirmed() - if err != nil { - log.Debugf("Get unconfirmed transactions failed, error %s", err.Error()) - return - } - now := time.Now() - for _, tx := range txs { - if now.After(tx.Timestamp.Add(MaxUnconfirmedTime)) { - err = w.db.Txs().Del(&tx.TxId) - if err != nil { - log.Errorf("Delete timeout transaction %s failed, error %s", tx.TxId.String(), err.Error()) - } + if !w.IsCurrent() { + return + } + + w.db.State().PutHeight(block.Height) + // Get all unconfirmed transactions + txs, err := w.db.Txs().GetAllUnconfirmed() + if err != nil { + log.Debugf("Get unconfirmed transactions failed, error %s", err.Error()) + return + } + now := time.Now() + for _, tx := range txs { + if now.After(tx.Timestamp.Add(MaxUnconfirmedTime)) { + err = w.db.Txs().Del(&tx.TxId) + if err != nil { + log.Errorf("Delete timeout transaction %s failed, error %s", tx.TxId.String(), err.Error()) } } } @@ -228,10 +195,10 @@ type txBatch struct { filter *sdk.AddrFilter } -// AddTx add a store transaction operation into batch, and return +// PutTx add a store transaction operation into batch, and return // if it is a false positive and error. -func (b *txBatch) AddTx(tx *util.Tx) (bool, error) { - // This AddTx in batch used by the ChainStore when storing a block. +func (b *txBatch) PutTx(tx *util.Tx) (bool, error) { + // This PutTx in batch is used by the ChainStore when storing a block. // That means this transaction has been confirmed, so we need to remove // it from unconfirmed list. And also, double spend transactions should // been removed from unconfirmed list as well. diff --git a/sync/config.go b/sync/config.go index b53e9b2..9bd4f4b 100644 --- a/sync/config.go +++ b/sync/config.go @@ -2,7 +2,9 @@ package sync import ( "github.com/elastos/Elastos.ELA.SPV/blockchain" + "github.com/elastos/Elastos.ELA/bloom" + "github.com/elastos/Elastos.ELA/core" ) const ( @@ -17,7 +19,8 @@ type Config struct { MinPeersForSync int MaxPeers int - UpdateFilter func() *bloom.Filter + UpdateFilter func() *bloom.Filter + TransactionAnnounce func(tx *core.Transaction) } func NewDefaultConfig(chain *blockchain.BlockChain, diff --git a/sync/manager.go b/sync/manager.go index b9831f9..dab59e8 100644 --- a/sync/manager.go +++ b/sync/manager.go @@ -332,13 +332,8 @@ func (sm *SyncManager) handleTxMsg(tmsg *txMsg) { delete(state.requestedTxns, txHash) delete(sm.requestedTxns, txHash) - fp, err := sm.cfg.Chain.CommitTx(tmsg.tx) - if err != nil { - log.Errorf("commit transaction error %v", err) - } - - if fp { - log.Debugf("Tx %s from Peer%d is a false positive.", txHash.String(), peer.ID()) + if sm.cfg.TransactionAnnounce != nil { + sm.cfg.TransactionAnnounce(tmsg.tx) } } @@ -419,7 +414,7 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { // Check false positive rate. fpRate := state.fpRate.Update(block, fps) if fpRate > fprate.DefaultFalsePositiveRate*10 { - log.Warnf("bloom filter false positive rate %f too high," + + log.Warnf("bloom filter false positive rate %f too high,"+ " disconnecting...", fpRate) peer.Disconnect() return diff --git a/util/block.go b/util/block.go index 6beea28..df9eb53 100644 --- a/util/block.go +++ b/util/block.go @@ -1,5 +1,7 @@ package util +import "github.com/elastos/Elastos.ELA/core" + // Block represent a block that stored in the // blockchain database. type Block struct { @@ -7,5 +9,5 @@ type Block struct { Header // Transactions of this block. - Transactions []*Tx + Transactions []*core.Transaction } diff --git a/util/header.go b/util/header.go index ffbec39..789bc89 100644 --- a/util/header.go +++ b/util/header.go @@ -2,9 +2,9 @@ package util import ( "bytes" - "github.com/elastos/Elastos.ELA.Utility/common" "math/big" + "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" ) From 791d2b2f13fe75de122b701371b4f41b2c88ac3a Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 18 Sep 2018 12:07:37 +0800 Subject: [PATCH 33/73] remove unconfirmed transaction logics for now --- spvwallet/txidcache.go | 44 ---------------- spvwallet/txidcache_test.go | 49 ----------------- spvwallet/wallet.go | 101 ++---------------------------------- 3 files changed, 5 insertions(+), 189 deletions(-) delete mode 100644 spvwallet/txidcache.go delete mode 100644 spvwallet/txidcache_test.go diff --git a/spvwallet/txidcache.go b/spvwallet/txidcache.go deleted file mode 100644 index 748a89b..0000000 --- a/spvwallet/txidcache.go +++ /dev/null @@ -1,44 +0,0 @@ -package spvwallet - -import ( - "github.com/elastos/Elastos.ELA.Utility/common" - "sync" -) - -type TxIdCache struct { - sync.Mutex - txIds map[common.Uint256]struct{} - index uint32 - txIdIndex []common.Uint256 -} - -func NewTxIdCache(capacity int) *TxIdCache { - return &TxIdCache{ - txIds: make(map[common.Uint256]struct{}), - txIdIndex: make([]common.Uint256, capacity), - } -} - -func (ic *TxIdCache) Add(txId common.Uint256) bool { - ic.Lock() - defer ic.Unlock() - - // Remove oldest txId - ic.index = ic.index % uint32(cap(ic.txIdIndex)) - delete(ic.txIds, ic.txIdIndex[ic.index]) - - // Add new txId - ic.txIds[txId] = struct{}{} - ic.txIdIndex[ic.index] = txId - - // Increase index - ic.index++ - return true -} - -func (ic *TxIdCache) Get(txId common.Uint256) bool { - ic.Lock() - defer ic.Unlock() - _, ok := ic.txIds[txId] - return ok -} diff --git a/spvwallet/txidcache_test.go b/spvwallet/txidcache_test.go deleted file mode 100644 index 754f2cc..0000000 --- a/spvwallet/txidcache_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package spvwallet - -import ( - "crypto/rand" - "github.com/elastos/Elastos.ELA.Utility/common" - "testing" -) - -func TestTxIdCache(t *testing.T) { - var cap = 100 - cache := NewTxIdCache(cap) - - var firstId common.Uint256 - var lastId common.Uint256 - for i := 0; i <= cap; i++ { - var txId common.Uint256 - rand.Read(txId[:]) - - if i == 0 { - firstId = txId - } - ok := cache.Add(txId, uint32(i)) - if !ok { - t.Errorf("Add txId failed at height %d", i) - } - - t.Logf("TxId added %s height %d", txId.String(), i) - - if len(cache.txIds) > cap { - t.Errorf("Cache overflow to size %d", len(cache.txIds)) - } - - t.Logf("Cache index %d size %d", cache.index, len(cache.txIds)) - - lastId = txId - } - - _, ok := cache.Get(firstId) - if !ok { - t.Errorf("Oldest txId not removed from cache") - } - - height, ok := cache.Get(lastId) - if ok { - t.Errorf("Duplicated txId added %s", lastId.String()) - } - - t.Logf("Find cached txId at height %d", height) -} diff --git a/spvwallet/wallet.go b/spvwallet/wallet.go index aa49129..018d1c7 100644 --- a/spvwallet/wallet.go +++ b/spvwallet/wallet.go @@ -1,8 +1,6 @@ package spvwallet import ( - "time" - "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" @@ -17,10 +15,8 @@ import ( ) const ( - MaxUnconfirmedTime = time.Minute * 30 - MaxTxIdCached = 1000 - MaxPeers = 12 - MinPeersForSync = 2 + MaxPeers = 12 + MinPeersForSync = 2 ) type Wallet struct { @@ -28,7 +24,6 @@ type Wallet struct { rpcServer *rpc.Server chainStore database.ChainStore db sqlite.DataStore - txIds *TxIdCache filter *sdk.AddrFilter } @@ -49,7 +44,6 @@ func (w *Wallet) Batch() database.TxBatch { return &txBatch{ db: w.db, batch: w.db.Batch(), - ids: w.txIds, filter: w.getAddrFilter(), } } @@ -133,13 +127,6 @@ func (w *Wallet) loadAddrFilter() *sdk.AddrFilter { // TransactionAnnounce will be invoked when received a new announced transaction. func (w *Wallet) TransactionAnnounce(tx *core.Transaction) { // TODO - // Save transaction as unconfirmed. - err := w.db.Txs().Put(sutil.NewTx(*tx, 0)) - if err != nil { - return - } - - w.txIds.Add(tx.Hash()) } // TransactionAccepted will be invoked after a transaction sent by @@ -153,14 +140,12 @@ func (w *Wallet) TransactionAccepted(tx *core.Transaction) { // method has been rejected. func (w *Wallet) TransactionRejected(tx *core.Transaction) { // TODO - } // TransactionConfirmed will be invoked after a transaction sent by // SendTransaction() method has been packed into a block. func (w *Wallet) TransactionConfirmed(tx *util.Tx) { // TODO - } // BlockCommitted will be invoked when a block and transactions within it are @@ -171,56 +156,22 @@ func (w *Wallet) BlockCommitted(block *util.Block) { } w.db.State().PutHeight(block.Height) - // Get all unconfirmed transactions - txs, err := w.db.Txs().GetAllUnconfirmed() - if err != nil { - log.Debugf("Get unconfirmed transactions failed, error %s", err.Error()) - return - } - now := time.Now() - for _, tx := range txs { - if now.After(tx.Timestamp.Add(MaxUnconfirmedTime)) { - err = w.db.Txs().Del(&tx.TxId) - if err != nil { - log.Errorf("Delete timeout transaction %s failed, error %s", tx.TxId.String(), err.Error()) - } - } - } + // TODO } type txBatch struct { db sqlite.DataStore batch sqlite.DataBatch - ids *TxIdCache filter *sdk.AddrFilter } // PutTx add a store transaction operation into batch, and return // if it is a false positive and error. func (b *txBatch) PutTx(tx *util.Tx) (bool, error) { - // This PutTx in batch is used by the ChainStore when storing a block. - // That means this transaction has been confirmed, so we need to remove - // it from unconfirmed list. And also, double spend transactions should - // been removed from unconfirmed list as well. txId := tx.Hash() height := tx.Height - dubs, err := b.checkDoubleSpends(tx) - if err != nil { - return false, nil - } - // Delete any double spend transactions - if len(dubs) > 0 { - batch := b.db.Txs().Batch() - for _, dub := range dubs { - if err := batch.Del(dub); err != nil { - batch.Rollback() - return false, nil - } - } - batch.Commit() - } - hits := 0 + // Check if any UTXOs within this wallet have been spent. for _, input := range tx.Inputs { // Move UTXO to STXO @@ -260,13 +211,11 @@ func (b *txBatch) PutTx(tx *util.Tx) (bool, error) { } // Save transaction - err = b.batch.Txs().Put(sutil.NewTx(tx.Transaction, height)) + err := b.batch.Txs().Put(sutil.NewTx(tx.Transaction, height)) if err != nil { return false, err } - b.ids.Add(txId) - return false, nil } @@ -293,43 +242,6 @@ func (b *txBatch) Commit() error { return b.batch.Commit() } -// checkDoubleSpends takes a transaction and compares it with all unconfirmed -// transactions in the db. It returns a slice of txIds in the db which are -// double spent by the received tx. -func (b *txBatch) checkDoubleSpends(tx *util.Tx) ([]*common.Uint256, error) { - txId := tx.Hash() - txs, err := b.db.Txs().GetAllUnconfirmed() - if err != nil { - return nil, err - } - - inputs := make(map[string]*common.Uint256) - for _, compTx := range txs { - // Skip coinbase transaction - if compTx.Data.IsCoinBaseTx() { - continue - } - - // Skip duplicate transaction - compTxId := compTx.Data.Hash() - if compTxId.IsEqual(txId) { - continue - } - - for _, in := range compTx.Data.Inputs { - inputs[in.ReferKey()] = &compTxId - } - } - - var dubs []*common.Uint256 - for _, in := range tx.Inputs { - if tx, ok := inputs[in.ReferKey()]; ok { - dubs = append(dubs, tx) - } - } - return dubs, nil -} - func New() (*Wallet, error) { wallet := new(Wallet) @@ -348,9 +260,6 @@ func New() (*Wallet, error) { // Initiate ChainStore wallet.chainStore = database.NewDefaultChainDB(headers, wallet) - // Initialize txs cache - wallet.txIds = NewTxIdCache(MaxTxIdCached) - // Initialize spv service wallet.IService, err = sdk.NewService( &sdk.Config{ From 1219d962094b42329c861b0b87b5be399929eea0 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 18 Sep 2018 18:47:18 +0800 Subject: [PATCH 34/73] add txProcessed and blockProcessed channal to spvservice instance --- sdk/spvservice.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/sdk/spvservice.go b/sdk/spvservice.go index 1adf0f7..84c19d4 100644 --- a/sdk/spvservice.go +++ b/sdk/spvservice.go @@ -50,6 +50,9 @@ type service struct { donePeers chan *peer.Peer txQueue chan interface{} quit chan struct{} + // The following chans are used to sync blockmanager and server. + txProcessed chan struct{} + blockProcessed chan struct{} } // Create a instance of SPV service implementation. @@ -67,6 +70,8 @@ func NewSPVService(cfg *Config) (*service, error) { donePeers: make(chan *peer.Peer, cfg.MaxPeers), txQueue: make(chan interface{}, 3), quit: make(chan struct{}), + txProcessed: make(chan struct{}, 1), + blockProcessed: make(chan struct{}, 1), } var maxPeers int @@ -403,11 +408,10 @@ func (s *service) onInv(sp *spvpeer.Peer, inv *msg.Inv) { } func (s *service) onBlock(sp *spvpeer.Peer, block *util.Block) { - done := make(chan struct{}) - s.syncManager.QueueBlock(block, sp, done) + s.syncManager.QueueBlock(block, sp, s.blockProcessed) select { - case <-done: + case <-s.blockProcessed: s.txQueue <- &blockMsg{block: block} if s.cfg.StateNotifier != nil { s.cfg.StateNotifier.BlockCommitted(block) @@ -415,10 +419,9 @@ func (s *service) onBlock(sp *spvpeer.Peer, block *util.Block) { } } -func (s *service) onTx(sp *spvpeer.Peer, tx *core.Transaction) { - done := make(chan struct{}) - s.syncManager.QueueTx(tx, sp, done) - <-done +func (s *service) onTx(sp *spvpeer.Peer, msgTx *core.Transaction) { + s.syncManager.QueueTx(msgTx, sp, s.txProcessed) + <-s.txProcessed } func (s *service) onNotFound(sp *spvpeer.Peer, notFound *msg.NotFound) { From 1db202b512dadc4ee993f68eb60da89c5f4c515f Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 20 Sep 2018 12:28:25 +0800 Subject: [PATCH 35/73] limit hashes count and flags size for header to prevent OOM --- util/header.go | 51 +++++++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/util/header.go b/util/header.go index 789bc89..3f91bd2 100644 --- a/util/header.go +++ b/util/header.go @@ -2,9 +2,11 @@ package util import ( "bytes" + "fmt" "math/big" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA/core" ) @@ -23,48 +25,49 @@ type Header struct { TotalWork *big.Int } -func (sh *Header) Serialize() ([]byte, error) { +func (h *Header) Serialize() ([]byte, error) { buf := new(bytes.Buffer) - err := sh.Header.Serialize(buf) + err := h.Header.Serialize(buf) if err != nil { return nil, err } - err = common.WriteUint32(buf, sh.NumTxs) + err = common.WriteUint32(buf, h.NumTxs) if err != nil { return nil, err } - err = common.WriteVarUint(buf, uint64(len(sh.Hashes))) + err = common.WriteVarUint(buf, uint64(len(h.Hashes))) if err != nil { return nil, err } - err = common.WriteElement(buf, sh.Hashes) - if err != nil { - return nil, err + for _, hash := range h.Hashes { + if err := hash.Serialize(buf); err != nil { + return nil, err + } } - err = common.WriteVarBytes(buf, sh.Flags) + err = common.WriteVarBytes(buf, h.Flags) if err != nil { return nil, err } - biBytes := sh.TotalWork.Bytes() + biBytes := h.TotalWork.Bytes() pad := make([]byte, 32-len(biBytes)) serializedBI := append(pad, biBytes...) buf.Write(serializedBI) return buf.Bytes(), nil } -func (sh *Header) Deserialize(b []byte) error { +func (h *Header) Deserialize(b []byte) error { r := bytes.NewReader(b) - err := sh.Header.Deserialize(r) + err := h.Header.Deserialize(r) if err != nil { return err } - sh.NumTxs, err = common.ReadUint32(r) + h.NumTxs, err = common.ReadUint32(r) if err != nil { return err } @@ -73,14 +76,24 @@ func (sh *Header) Deserialize(b []byte) error { if err != nil { return err } + if count > msg.MaxTxPerBlock { + str := fmt.Sprintf("too many transactions to fit into a block "+ + "[count %d, max %d]", count, msg.MaxTxPerBlock) + return common.FuncError("Header.Deserialize", str) + } - sh.Hashes = make([]*common.Uint256, count) - err = common.ReadElement(r, &sh.Hashes) - if err != nil { - return err + hashes := make([]common.Uint256, count) + h.Hashes = make([]*common.Uint256, 0, count) + for i := uint64(0); i < count; i++ { + hash := &hashes[i] + if err := hash.Deserialize(r); err != nil { + return err + } + h.Hashes = append(h.Hashes, hash) } - sh.Flags, err = common.ReadVarBytes(r) + h.Flags, err = common.ReadVarBytes(r, msg.MaxTxPerBlock, + "header merkle proof flags") if err != nil { return err } @@ -90,8 +103,8 @@ func (sh *Header) Deserialize(b []byte) error { if err != nil { return err } - sh.TotalWork = new(big.Int) - sh.TotalWork.SetBytes(biBytes) + h.TotalWork = new(big.Int) + h.TotalWork.SetBytes(biBytes) return nil } From 5237dad2c2fc0f3cd54c71f07b9672153798bc98 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Fri, 21 Sep 2018 18:54:27 +0800 Subject: [PATCH 36/73] minor fix for Utility server interface change --- sdk/spvservice.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/spvservice.go b/sdk/spvservice.go index 84c19d4..3dd401b 100644 --- a/sdk/spvservice.go +++ b/sdk/spvservice.go @@ -178,14 +178,14 @@ func (s *service) makeEmptyMessage(cmd string) (p2p.Message, error) { return message, nil } -func (s *service) newPeer(peer *peer.Peer) { +func (s *service) newPeer(peer server.IPeer) { log.Debugf("server new peer %v", peer) - s.newPeers <- peer + s.newPeers <- peer.ToPeer() } -func (s *service) donePeer(peer *peer.Peer) { +func (s *service) donePeer(peer server.IPeer) { log.Debugf("server done peer %v", peer) - s.donePeers <- peer + s.donePeers <- peer.ToPeer() } // peerHandler handles new peers and done peers from P2P server. From 534e6a1498f07fb9fd8ee4f66e0e8ffbf1df0b28 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 27 Sep 2018 12:15:45 +0800 Subject: [PATCH 37/73] remove unused makeEmptyMessage cases for thay have implemented by peer it's self --- sdk/spvservice.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/sdk/spvservice.go b/sdk/spvservice.go index 3dd401b..2e4fc4d 100644 --- a/sdk/spvservice.go +++ b/sdk/spvservice.go @@ -136,18 +136,6 @@ func (s *service) updateFilter() *bloom.Filter { func (s *service) makeEmptyMessage(cmd string) (p2p.Message, error) { var message p2p.Message switch cmd { - case p2p.CmdVersion: - message = new(msg.Version) - - case p2p.CmdVerAck: - message = new(msg.VerAck) - - case p2p.CmdGetAddr: - message = new(msg.GetAddr) - - case p2p.CmdAddr: - message = new(msg.Addr) - case p2p.CmdInv: message = new(msg.Inv) @@ -160,12 +148,6 @@ func (s *service) makeEmptyMessage(cmd string) (p2p.Message, error) { case p2p.CmdTx: message = msg.NewTx(new(core.Transaction)) - case p2p.CmdPing: - message = new(msg.Ping) - - case p2p.CmdPong: - message = new(msg.Pong) - case p2p.CmdMerkleBlock: message = msg.NewMerkleBlock(new(core.Header)) From 712fecad76febfd78aaba3574932651860c7286e Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 29 Sep 2018 15:08:22 +0800 Subject: [PATCH 38/73] minor fix for sdk package, make service privite --- sdk/interface.go | 3 ++- sdk/{spvservice.go => service.go} | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) rename sdk/{spvservice.go => service.go} (99%) diff --git a/sdk/interface.go b/sdk/interface.go index a3ead57..c9f8578 100644 --- a/sdk/interface.go +++ b/sdk/interface.go @@ -3,6 +3,7 @@ package sdk import ( "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" ) @@ -99,5 +100,5 @@ DataStore is an interface including all methods you need to implement placed in Also an sample APP spvwallet is contain in this project placed in spvwallet folder. */ func NewService(config *Config) (IService, error) { - return NewSPVService(config) + return newService(config) } diff --git a/sdk/spvservice.go b/sdk/service.go similarity index 99% rename from sdk/spvservice.go rename to sdk/service.go index 2e4fc4d..b9bac09 100644 --- a/sdk/spvservice.go +++ b/sdk/service.go @@ -56,7 +56,7 @@ type service struct { } // Create a instance of SPV service implementation. -func NewSPVService(cfg *Config) (*service, error) { +func newService(cfg *Config) (*service, error) { // Initialize blockchain chain, err := blockchain.New(cfg.Foundation, cfg.ChainStore) if err != nil { From 6956d26b7ab63f4094c1f5be1afb8193ebe9d30d Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 29 Sep 2018 15:14:03 +0800 Subject: [PATCH 39/73] adjust spvservice interface, add Stop() method to spvservice --- interface/interface.go | 5 ++++- interface/spvservice.go | 23 ----------------------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/interface/interface.go b/interface/interface.go index 62be03a..0bbb091 100644 --- a/interface/interface.go +++ b/interface/interface.go @@ -61,7 +61,10 @@ type SPVService interface { HeaderStore() database.Headers // Start the SPV service - Start() error + Start() + + // Stop the SPV service + Stop() // ClearData delete all data stores data including HeaderStore and DataStore. ClearData() error diff --git a/interface/spvservice.go b/interface/spvservice.go index 67cc84d..339cd39 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -5,8 +5,6 @@ import ( "crypto/sha256" "errors" "fmt" - "os" - "os/signal" "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/interface/store" @@ -229,27 +227,6 @@ func (s *spvservice) BlockCommitted(block *util.Block) { } } -func (s *spvservice) Start() error { - // Handle interrupt signal - quit := make(chan struct{}) - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt) - go func() { - for range signals { - log.Trace("SPV service shutting down...") - s.IService.Stop() - quit <- struct{}{} - } - }() - - // Start SPV service - s.IService.Start() - - <-quit - - return nil -} - func (s *spvservice) ClearData() error { if err := s.headers.Clear(); err != nil { log.Warnf("Clear header store error %s", err.Error()) From b4ef52112b3646a8e16502a362301c1db4af65cd Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 9 Oct 2018 11:06:57 +0800 Subject: [PATCH 40/73] fix compile issue for Utility source code change --- sdk/service.go | 1 + sync/manager.go | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sdk/service.go b/sdk/service.go index b9bac09..fd1f15e 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -102,6 +102,7 @@ func newService(cfg *Config) (*service, error) { // Initiate P2P server configuration serverCfg := server.NewDefaultConfig( cfg.Magic, + OpenService, cfg.DefaultPort, cfg.SeedList, nil, diff --git a/sync/manager.go b/sync/manager.go index dab59e8..a4519d2 100644 --- a/sync/manager.go +++ b/sync/manager.go @@ -9,7 +9,6 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA/core" ) @@ -216,13 +215,14 @@ func (sm *SyncManager) syncWith(p *peer.Peer) { // isSyncCandidate returns whether or not the peer is a candidate to consider // syncing from. func (sm *SyncManager) isSyncCandidate(peer *peer.Peer) bool { + // TODO implement this when main chain refactor completed. + //services := peer.Services() + //// Candidate if all checks passed. + //return services&p2p.SFNodeNetwork == p2p.SFNodeNetwork && + // services&p2p.SFNodeBloom == p2p.SFNodeBloom + // Just return true. return true - - services := peer.Services() - // Candidate if all checks passed. - return services&p2p.SFNodeNetwork == p2p.SFNodeNetwork && - services&p2p.SFNodeBloom == p2p.SFNodeBloom } // getSyncCandidates returns the peers that are sync candidate. From 177d71c31bb147a629448ac750c6a4b4b398d734 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 11 Oct 2018 15:02:21 +0800 Subject: [PATCH 41/73] remove GenesisHeader() method from sdk/protocol.go and do code format --- interface/interface.go | 2 +- interface/interface_test.go | 29 ++++++++++-- interface/spvservice.go | 2 +- interface/store/databatch.go | 8 ++-- interface/store/opsbatch.go | 2 +- sdk/protocal.go | 84 ++------------------------------- spvwallet/store/sqlite/stxos.go | 3 +- spvwallet/sutil/tx.go | 5 +- 8 files changed, 43 insertions(+), 92 deletions(-) diff --git a/interface/interface.go b/interface/interface.go index 0bbb091..b06dd5f 100644 --- a/interface/interface.go +++ b/interface/interface.go @@ -100,5 +100,5 @@ type TransactionListener interface { } func NewSPVService(config *Config) (SPVService, error) { - return newService(config) + return newSpvService(config) } diff --git a/interface/interface_test.go b/interface/interface_test.go index d934112..d5b4a7e 100644 --- a/interface/interface_test.go +++ b/interface/interface_test.go @@ -3,6 +3,7 @@ package _interface import ( "fmt" "testing" + "time" "github.com/elastos/Elastos.ELA.SPV/blockchain" spvpeer "github.com/elastos/Elastos.ELA.SPV/peer" @@ -120,7 +121,7 @@ func TestNewSPVService(t *testing.T) { MaxConnections: 100, } - service, err := NewSPVService(config) + service, err := newSpvService(config) if err != nil { t.Error("NewSPVService error %s", err.Error()) } @@ -142,8 +143,28 @@ func TestNewSPVService(t *testing.T) { service.RegisterTransactionListener(unconfirmedListener) // Start spv service - err = service.Start() - if err != nil { - t.Error("Start SPV service error: ", err) + service.Start() + + syncTicker := time.NewTicker(time.Second * 10) + defer syncTicker.Stop() + +out: + for { + select { + case <-syncTicker.C: + + if service.IService.IsCurrent() { + // Clear test data + err := service.ClearData() + if err != nil { + t.Errorf("service clear data error %s", err) + } + + service.Stop() + t.Log("successful synchronized to current") + + break out + } + } } } diff --git a/interface/spvservice.go b/interface/spvservice.go index 339cd39..8e62894 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -27,7 +27,7 @@ type spvservice struct { listeners map[common.Uint256]TransactionListener } -func newService(cfg *Config) (*spvservice, error) { +func newSpvService(cfg *Config) (*spvservice, error) { headerStore, err := store.NewHeaderStore() if err != nil { return nil, err diff --git a/interface/store/databatch.go b/interface/store/databatch.go index 5436824..81ea6ec 100644 --- a/interface/store/databatch.go +++ b/interface/store/databatch.go @@ -5,11 +5,13 @@ import ( "database/sql" "encoding/binary" "encoding/gob" - "github.com/boltdb/bolt" + "sync" + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/boltdb/bolt" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" - "sync" ) // Ensure dataBatch implement DataBatch interface. @@ -69,7 +71,7 @@ func (b *dataBatch) DelAll(height uint32) error { } } - err = b.boltTx.Bucket(BKTHeightTxs).Delete(key[:]) + err = b.boltTx.Bucket(BKTHeightTxs).Delete(key[:]) if err != nil { return err } diff --git a/interface/store/opsbatch.go b/interface/store/opsbatch.go index 2a6bbc9..d5c6892 100644 --- a/interface/store/opsbatch.go +++ b/interface/store/opsbatch.go @@ -1,9 +1,9 @@ package store import ( - "github.com/boltdb/bolt" "sync" + "github.com/boltdb/bolt" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" ) diff --git a/sdk/protocal.go b/sdk/protocal.go index fa24e89..4536feb 100644 --- a/sdk/protocal.go +++ b/sdk/protocal.go @@ -1,87 +1,13 @@ package sdk import ( - "time" - - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/crypto" "github.com/elastos/Elastos.ELA.Utility/p2p" - - ela "github.com/elastos/Elastos.ELA/core" ) const ( - ProtocolVersion = p2p.EIP001Version // The protocol version implemented SPV protocol - OpenService = 1 << 2 -) - -func GenesisHeader(foundation *common.Uint168) *ela.Header { - // header - header := ela.Header{ - Version: ela.BlockVersion, - Previous: common.EmptyHash, - MerkleRoot: common.EmptyHash, - Timestamp: uint32(time.Unix(time.Date(2017, time.December, 22, 10, 0, 0, 0, time.UTC).Unix(), 0).Unix()), - Bits: 0x1d03ffff, - Nonce: ela.GenesisNonce, - Height: uint32(0), - } - - // ELA coin - elaCoin := &ela.Transaction{ - TxType: ela.RegisterAsset, - PayloadVersion: 0, - Payload: &ela.PayloadRegisterAsset{ - Asset: ela.Asset{ - Name: "ELA", - Precision: 0x08, - AssetType: 0x00, - }, - Amount: 0 * 100000000, - Controller: common.Uint168{}, - }, - Attributes: []*ela.Attribute{}, - Inputs: []*ela.Input{}, - Outputs: []*ela.Output{}, - Programs: []*ela.Program{}, - } + // The protocol version implemented SPV protocol + ProtocolVersion = p2p.EIP001Version - coinBase := &ela.Transaction{ - TxType: ela.CoinBase, - PayloadVersion: ela.PayloadCoinBaseVersion, - Payload: new(ela.PayloadCoinBase), - Inputs: []*ela.Input{ - { - Previous: ela.OutPoint{ - TxID: common.EmptyHash, - Index: 0x0000, - }, - Sequence: 0x00000000, - }, - }, - Attributes: []*ela.Attribute{}, - LockTime: 0, - Programs: []*ela.Program{}, - } - - coinBase.Outputs = []*ela.Output{ - { - AssetID: elaCoin.Hash(), - Value: 3300 * 10000 * 100000000, - ProgramHash: *foundation, - }, - } - - nonce := []byte{0x4d, 0x65, 0x82, 0x21, 0x07, 0xfc, 0xfd, 0x52} - txAttr := ela.NewAttribute(ela.Nonce, nonce) - coinBase.Attributes = append(coinBase.Attributes, &txAttr) - - transactions := []*ela.Transaction{coinBase, elaCoin} - hashes := make([]common.Uint256, 0, len(transactions)) - for _, tx := range transactions { - hashes = append(hashes, tx.Hash()) - } - header.MerkleRoot, _ = crypto.ComputeRoot(hashes) - - return &header -} + // OpenService is a flag used to indicate a peer provides open service. + OpenService = 1 << 2 +) diff --git a/spvwallet/store/sqlite/stxos.go b/spvwallet/store/sqlite/stxos.go index 4c84fb7..3b6043f 100644 --- a/spvwallet/store/sqlite/stxos.go +++ b/spvwallet/store/sqlite/stxos.go @@ -2,9 +2,10 @@ package sqlite import ( "database/sql" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" "sync" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" ) diff --git a/spvwallet/sutil/tx.go b/spvwallet/sutil/tx.go index 6fb0c87..1d06b0e 100644 --- a/spvwallet/sutil/tx.go +++ b/spvwallet/sutil/tx.go @@ -1,10 +1,11 @@ package sutil import ( + "fmt" "time" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA.Utility/common" - "fmt" + "github.com/elastos/Elastos.ELA/core" ) type Tx struct { From f9fe8e303e6ce88f64f45aec8718747ec2a3b484 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 11 Oct 2018 16:46:22 +0800 Subject: [PATCH 42/73] add MinPeersForSync parameter for interface/spvservice --- interface/interface.go | 3 +++ interface/spvservice.go | 4 +--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/interface/interface.go b/interface/interface.go index b06dd5f..1419bab 100644 --- a/interface/interface.go +++ b/interface/interface.go @@ -23,6 +23,9 @@ type Config struct { // DefaultPort is the default port for public peers provide services. DefaultPort uint16 + // The min candidate peers count to start syncing progress. + MinPeersForSync int + // The minimum target outbound connections. MinOutbound int diff --git a/interface/spvservice.go b/interface/spvservice.go index 8e62894..d51f1b2 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -17,8 +17,6 @@ import ( "github.com/elastos/Elastos.ELA/core" ) -const minPeersForSync = 3 - type spvservice struct { sdk.IService headers store.HeaderStore @@ -52,7 +50,7 @@ func newSpvService(cfg *Config) (*spvservice, error) { SeedList: cfg.SeedList, DefaultPort: cfg.DefaultPort, MaxPeers: cfg.MaxConnections, - MinPeersForSync: minPeersForSync, + MinPeersForSync: cfg.MinPeersForSync, Foundation: cfg.Foundation, ChainStore: chainStore, GetFilterData: service.GetFilterData, From 61e19e2b0e843bf4663fdda87e6614a4dcc32df6 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 15 Oct 2018 11:47:41 +0800 Subject: [PATCH 43/73] use Utility elalog/ package instead log/ package --- Makefile | 2 +- log.go | 59 ++++++++ log/log.go | 379 ------------------------------------------------ log/log_test.go | 27 ---- main.go | 29 +--- peer/log.go | 4 + sdk/service.go | 1 + 7 files changed, 67 insertions(+), 434 deletions(-) create mode 100644 log.go delete mode 100644 log/log.go delete mode 100644 log/log_test.go diff --git a/Makefile b/Makefile index 2faab45..2948f28 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ BUILD=go build VERSION := $(shell git describe --abbrev=4 --dirty --always --tags) BUILD_SPV_CLI =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o ela-wallet client.go -BUILD_SPV_SERVICE =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o service main.go +BUILD_SPV_SERVICE =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o service log.go main.go all: $(BUILD_SPV_CLI) diff --git a/log.go b/log.go new file mode 100644 index 0000000..405685d --- /dev/null +++ b/log.go @@ -0,0 +1,59 @@ +package main + +import ( + "io" + "os" + + "github.com/elastos/Elastos.ELA.SPV/blockchain" + "github.com/elastos/Elastos.ELA.SPV/peer" + "github.com/elastos/Elastos.ELA.SPV/sdk" + "github.com/elastos/Elastos.ELA.SPV/spvwallet" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" + "github.com/elastos/Elastos.ELA.SPV/spvwallet/store" + "github.com/elastos/Elastos.ELA.SPV/sync" + + "github.com/elastos/Elastos.ELA.Utility/elalog" + "github.com/elastos/Elastos.ELA.Utility/p2p/addrmgr" + "github.com/elastos/Elastos.ELA.Utility/p2p/connmgr" + "github.com/elastos/Elastos.ELA.Utility/p2p/server" +) + +const LogPath = "./logs-spv/" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var ( + fileWriter = elalog.NewFileWriter( + LogPath, + config.Values().MaxPerLogSize, + config.Values().MaxLogsSize, + ) + level = elalog.Level(config.Values().PrintLevel) + backend = elalog.NewBackend(io.MultiWriter(os.Stdout, fileWriter), + elalog.Llongfile) + + admrlog = backend.Logger("ADMR", elalog.LevelOff) + cmgrlog = backend.Logger("CMGR", elalog.LevelOff) + bcdblog = backend.Logger("BCDB", level) + synclog = backend.Logger("SYNC", level) + peerlog = backend.Logger("PEER", level) + spvslog = backend.Logger("SPVS", level) + srvrlog = backend.Logger("SRVR", level) + rpcslog = backend.Logger("RPCS", level) + waltlog = backend.Logger("WALT", level) +) + +func init() { + addrmgr.UseLogger(admrlog) + connmgr.UseLogger(cmgrlog) + blockchain.UseLogger(bcdblog) + sdk.UseLogger(spvslog) + rpc.UseLogger(rpcslog) + peer.UseLogger(peerlog) + server.UseLogger(srvrlog) + store.UseLogger(bcdblog) + sync.UseLogger(synclog) + spvwallet.UseLogger(waltlog) +} diff --git a/log/log.go b/log/log.go deleted file mode 100644 index 6b0871c..0000000 --- a/log/log.go +++ /dev/null @@ -1,379 +0,0 @@ -package log - -import ( - "bytes" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "path/filepath" - "runtime" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/AlexpanXX/fsnotify" -) - -const ( - White = "1;00" - Blue = "1;34" - Red = "1;31" - Green = "1;32" - Yellow = "1;33" - Cyan = "1;36" - Pink = "1;35" -) - -func Color(code, msg string) string { - return fmt.Sprintf("\033[%sm%s\033[m", code, msg) -} - -const ( - infoLog = iota - warnLog - errorLog - fatalLog - traceLog - debugLog - maxLevelLog -) - -var ( - levels = map[int]string{ - infoLog: Color(White, "[INFO]"), - warnLog: Color(Yellow, "[WARN]"), - errorLog: Color(Red, "[ERROR]"), - fatalLog: Color(Pink, "[FATAL]"), - traceLog: Color(Cyan, "[TRACE]"), - debugLog: Color(Green, "[DEBUG]"), - } -) - -const ( - namePrefix = "LEVEL" - callDepth = 2 - KB_SIZE = int64(1024) - MB_SIZE = KB_SIZE * 1024 - GB_SIZE = MB_SIZE * 1024 - defaultMaxPerLogSize = 20 * MB_SIZE - defaultMaxLogsSize = 5 * GB_SIZE -) - -func GetGID() uint64 { - var buf [64]byte - b := buf[:runtime.Stack(buf[:], false)] - b = bytes.TrimPrefix(b, []byte("goroutine ")) - b = b[:bytes.IndexByte(b, ' ')] - n, _ := strconv.ParseUint(string(b), 10, 64) - return n -} - -func LevelName(level int) string { - if name, ok := levels[level]; ok { - return name - } - return namePrefix + strconv.Itoa(level) -} - -type Logger struct { - path string - level int // The log print level - maxLogsSize int64 // The max logs total size - - // Current log file and printer - mutex sync.Mutex - maxPerLogSize int64 - file *os.File - logger *log.Logger - watcher *fsnotify.Watcher -} - -func (l *Logger) Level() string { - return LevelName(l.level) -} - -func (l *Logger) init() { - // setup file watcher for the printing log file, - // watch the file size change and trigger new log - // file create when the MaxPerLogSize limit reached. - var err error - l.watcher, err = fsnotify.NewWatcher() - if err != nil { - fmt.Println("create log file watcher failed,", err) - os.Exit(-1) - } - go func() { - for { - select { - case event := <-l.watcher.Events: - l.handleFileEvents(event) - case err := <-l.watcher.Errors: - fmt.Println("error:", err.Error()) - } - } - }() - - // create new log file - l.newLogFile() -} - -func (l *Logger) prune() { - // load the file list under logs output path - // so we can delete the oldest log file when - // the MaxLogsSize limit reached. - fileList, err := ioutil.ReadDir(l.path) - if err != nil { - fmt.Println("read logs path failed,", err) - } - SortLogFiles(fileList) - - // calculate total size - var totalSize int64 - for _, f := range fileList { - totalSize += f.Size() - } - for totalSize >= l.maxLogsSize { - // Get the oldest log file - file := fileList[0] - // Remove it - os.Remove(l.path + file.Name()) - fileList = fileList[1:] - // Update logs size - totalSize -= file.Size() - } -} - -func (l *Logger) newLogFile() { - // prune before create a new log file - l.prune() - - // create new log file - var err error - l.file, err = newLogFile(l.path) - if err != nil { - fmt.Print("create log file failed,", err.Error()) - os.Exit(-1) - } - - // get file stat - info, err := l.file.Stat() - if err != nil { - fmt.Print("get log file stat failed,", err.Error()) - os.Exit(-1) - } - - // setup new printer - l.logger = log.New(io.MultiWriter(os.Stdout, l.file), "", log.Ldate|log.Lmicroseconds) - - // watch log file change - l.watcher.Add(l.path + info.Name()) -} - -func (l *Logger) handleFileEvents(event fsnotify.Event) { - switch event.Op { - case fsnotify.Write: - info, _ := l.file.Stat() - if info.Size() >= l.maxPerLogSize { - l.mutex.Lock() - // close previous log file - l.file.Close() - // unwatch it - l.watcher.Remove(l.path + info.Name()) - // create a new log file - l.newLogFile() - l.mutex.Unlock() - } - } -} - -func NewLogger(path string, level int, maxPerLogSizeMb, maxLogsSizeMb int64) *Logger { - logger := new(Logger) - logger.path = path - logger.level = level - - if maxPerLogSizeMb != 0 { - logger.maxPerLogSize = maxPerLogSizeMb * MB_SIZE - } else { - logger.maxPerLogSize = defaultMaxPerLogSize - } - - if maxLogsSizeMb != 0 { - logger.maxLogsSize = maxLogsSizeMb * MB_SIZE - } else { - logger.maxLogsSize = defaultMaxLogsSize - } - - logger.init() - return logger -} - -func newLogFile(path string) (*os.File, error) { - if fi, err := os.Stat(path); err == nil { - if !fi.IsDir() { - return nil, fmt.Errorf("open %s: not a directory", path) - } - } else if os.IsNotExist(err) { - if err := os.MkdirAll(path, 0766); err != nil { - return nil, err - } - } else { - return nil, err - } - - var timestamp = time.Now().Format("2006-01-02_15.04.05") - - file, err := os.OpenFile(path+timestamp+"_LOG.log", os.O_RDWR|os.O_CREATE, 0666) - if err != nil { - return nil, err - } - return file, nil -} - -func SortLogFiles(files []os.FileInfo) { - sort.Sort(byTime(files)) -} - -type byTime []os.FileInfo - -func (f byTime) Len() int { return len(f) } -func (f byTime) Less(i, j int) bool { return f[i].Name() < f[j].Name() } -func (f byTime) Swap(i, j int) { f[i], f[j] = f[j], f[i] } - -func (l *Logger) SetPrintLevel(level int) error { - if level > maxLevelLog || level < 0 { - return errors.New("Invalid Debug Level") - } - - l.level = level - return nil -} - -func (l *Logger) Output(level int, a ...interface{}) error { - l.mutex.Lock() - defer l.mutex.Unlock() - if l.level >= level { - gidStr := strconv.FormatUint(GetGID(), 10) - a = append([]interface{}{LevelName(level), "GID", gidStr + ","}, a...) - return l.logger.Output(callDepth, fmt.Sprintln(a...)) - } - return nil -} - -func (l *Logger) Outputf(level int, format string, v ...interface{}) error { - l.mutex.Lock() - defer l.mutex.Unlock() - if l.level >= level { - v = append([]interface{}{LevelName(level), "GID", GetGID()}, v...) - return l.logger.Output(callDepth, fmt.Sprintf("%s %s %d, "+format+"\n", v...)) - } - return nil -} - -func (l *Logger) Trace(a ...interface{}) { - if l.level < traceLog { - return - } - - pc := make([]uintptr, 10) - runtime.Callers(2, pc) - f := runtime.FuncForPC(pc[0]) - file, line := f.FileLine(pc[0]) - fileName := filepath.Base(file) - - nameFull := f.Name() - nameEnd := filepath.Ext(nameFull) - funcName := strings.TrimPrefix(nameEnd, ".") - - a = append([]interface{}{funcName + "()", fileName + ":" + strconv.Itoa(line)}, a...) - - l.Output(traceLog, a...) -} - -func (l *Logger) Tracef(format string, a ...interface{}) { - if l.level < traceLog { - return - } - - pc := make([]uintptr, 10) - runtime.Callers(2, pc) - f := runtime.FuncForPC(pc[0]) - file, line := f.FileLine(pc[0]) - fileName := filepath.Base(file) - - nameFull := f.Name() - nameEnd := filepath.Ext(nameFull) - funcName := strings.TrimPrefix(nameEnd, ".") - - a = append([]interface{}{funcName, fileName, line}, a...) - - l.Outputf(traceLog, "%s() %s:%d "+format, a...) -} - -func (l *Logger) Debug(a ...interface{}) { - if l.level < debugLog { - return - } - - pc := make([]uintptr, 10) - runtime.Callers(2, pc) - f := runtime.FuncForPC(pc[0]) - file, line := f.FileLine(pc[0]) - fileName := filepath.Base(file) - - a = append([]interface{}{f.Name(), fileName + ":" + strconv.Itoa(line)}, a...) - - l.Output(debugLog, a...) -} - -func (l *Logger) Debugf(format string, a ...interface{}) { - if l.level < debugLog { - return - } - - pc := make([]uintptr, 10) - runtime.Callers(2, pc) - f := runtime.FuncForPC(pc[0]) - file, line := f.FileLine(pc[0]) - fileName := filepath.Base(file) - - a = append([]interface{}{f.Name(), fileName, line}, a...) - - l.Outputf(debugLog, "%s %s:%d "+format, a...) -} - -func (l *Logger) Info(a ...interface{}) { - l.Output(infoLog, a...) -} - -func (l *Logger) Infof(format string, a ...interface{}) { - l.Outputf(infoLog, format, a...) -} - -func (l *Logger) Warn(a ...interface{}) { - l.Output(warnLog, a...) -} - -func (l *Logger) Warnf(format string, a ...interface{}) { - l.Outputf(warnLog, format, a...) -} - -func (l *Logger) Error(a ...interface{}) { - l.Output(errorLog, a...) -} - -func (l *Logger) Errorf(format string, a ...interface{}) { - l.Outputf(errorLog, format, a...) -} - -func (l *Logger) Fatal(a ...interface{}) { - l.Output(fatalLog, a...) -} - -func (l *Logger) Fatalf(format string, a ...interface{}) { - l.Outputf(fatalLog, format, a...) -} diff --git a/log/log_test.go b/log/log_test.go deleted file mode 100644 index 509780b..0000000 --- a/log/log_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package log - -import ( - "testing" - "time" -) - -func TestNewLogger(t *testing.T) { - logger := NewLogger("./log-test",0, 5, 20) - start := time.Now() - for { - logger.Info("Print info log") - logger.Infof("Print info log formatted") - logger.Trace("Print trace log") - logger.Tracef("Print trace log formatted") - logger.Warn("Print warn log") - logger.Warnf("Print warn log formatted") - logger.Error("Print error log") - logger.Errorf("Print error log formatted") - logger.Debug("Print debug log") - logger.Debugf("Print debug log formatted") - - if start.Add(time.Second * 15).Before(time.Now()) { - break - } - } -} diff --git a/main.go b/main.go index f8409fc..aa0f6dd 100644 --- a/main.go +++ b/main.go @@ -4,39 +4,14 @@ import ( "os" "os/signal" - "github.com/elastos/Elastos.ELA.SPV/blockchain" - "github.com/elastos/Elastos.ELA.SPV/log" - "github.com/elastos/Elastos.ELA.SPV/peer" - "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/spvwallet" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/store" - "github.com/elastos/Elastos.ELA.SPV/sync" ) -const LogPath = "./Logs-spv/" - func main() { - // Initiate logger - logger := log.NewLogger(LogPath, - config.Values().PrintLevel, - config.Values().MaxPerLogSize, - config.Values().MaxLogsSize, - ) - - sdk.UseLogger(logger) - rpc.UseLogger(logger) - peer.UseLogger(logger) - blockchain.UseLogger(logger) - store.UseLogger(logger) - sync.UseLogger(logger) - spvwallet.UseLogger(logger) - // Initiate SPV service wallet, err := spvwallet.New() if err != nil { - logger.Error("Initiate SPV service failed,", err) + waltlog.Error("Initiate SPV service failed,", err) os.Exit(0) } @@ -46,7 +21,7 @@ func main() { signal.Notify(c, os.Interrupt) go func() { for range c { - logger.Trace("Wallet shutting down...") + waltlog.Trace("Wallet shutting down...") wallet.Stop() stop <- 1 } diff --git a/peer/log.go b/peer/log.go index eddf96c..1993995 100644 --- a/peer/log.go +++ b/peer/log.go @@ -2,6 +2,7 @@ package peer import ( "github.com/elastos/Elastos.ELA.Utility/elalog" + "github.com/elastos/Elastos.ELA.Utility/p2p/peer" ) // log is a logger that is initialized with no output filters. This @@ -25,4 +26,7 @@ func DisableLog() { // using elalog. func UseLogger(logger elalog.Logger) { log = logger + + // set parent logger. + peer.UseLogger(logger) } diff --git a/sdk/service.go b/sdk/service.go index fd1f15e..b02b7bb 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -102,6 +102,7 @@ func newService(cfg *Config) (*service, error) { // Initiate P2P server configuration serverCfg := server.NewDefaultConfig( cfg.Magic, + p2p.EIP001Version, OpenService, cfg.DefaultPort, cfg.SeedList, From 56c9ea1d47fbda3302ca1a71bdfc303e1c1f8365 Mon Sep 17 00:00:00 2001 From: jiangzehua Date: Fri, 19 Oct 2018 16:45:41 +0800 Subject: [PATCH 44/73] queue message when commit block --- interface/spvservice.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/interface/spvservice.go b/interface/spvservice.go index d51f1b2..c95986f 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -189,6 +189,13 @@ func (s *spvservice) TransactionConfirmed(tx *util.Tx) {} // BlockCommitted will be invoked when a block and transactions within it are // successfully committed into database. func (s *spvservice) BlockCommitted(block *util.Block) { + log.Infof("Receive block %s height %d", block.Hash(), block.Height) + for _, tx := range block.Transactions { + for _, listener := range s.listeners { + s.queueMessageByListener(listener, tx, block.Height) + } + } + // Look up for queued transactions items, err := s.db.Que().GetAll() if err != nil { @@ -220,7 +227,7 @@ func (s *spvservice) BlockCommitted(block *util.Block) { Flags: block.Flags, }, storeTx.Transaction, - header.Height-item.Height, + block.Height-item.Height, ) } } From 891f832ec5d0f0e90a970f13f15102ac5b18db7d Mon Sep 17 00:00:00 2001 From: jiangzehua Date: Fri, 19 Oct 2018 18:46:40 +0800 Subject: [PATCH 45/73] modify glide file --- glide.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/glide.yaml b/glide.yaml index 03dd8f6..fc21b9d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -8,7 +8,9 @@ import: - package: github.com/boltdb/bolt - package: github.com/cevaris/ordered_map - package: github.com/elastos/Elastos.ELA.Utility + version: release_v0.1.1 - package: github.com/elastos/Elastos.ELA + version: release_v0.2.1 subpackages: - bloom - core From cd9425516faa981d46a83ca11bcd762edbfe8d61 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Fri, 19 Oct 2018 19:21:33 +0800 Subject: [PATCH 46/73] make SPV sdk and wallet compatible for different blockchains --- Makefile | 4 +- blockchain/{chain.go => blockchain.go} | 29 +- blockchain/difficulty.go | 4 +- bloom/bloom.go | 206 +++++++++ bloom/filter.go | 296 +++++++++++++ bloom/merkleproof.go | 85 ++++ bloom/murmurhash3.go | 72 ++++ client.go | 48 ++- config.go | 84 ++++ database/defaultdb.go | 2 +- database/txsdb.go | 2 +- .../genesis.go => interface/blockheader.go | 32 +- interface/interface.go | 4 +- interface/interface_test.go | 55 ++- interface/keystoreimpl.go | 6 +- interface/spvservice.go | 72 +++- interface/store/databatch.go | 14 +- interface/store/headers.go | 27 +- interface/store/interface.go | 11 +- interface/store/ops.go | 11 +- interface/store/opsbatch.go | 7 +- interface/store/txs.go | 4 +- interface/store/txsbatch.go | 6 +- log.go | 17 +- main.go | 8 +- peer/peer.go | 53 +-- sdk/bloom.go | 46 -- sdk/interface.go | 21 +- sdk/service.go | 53 ++- spvwallet.go | 399 ++++++++++++++++++ spvwallet/config/config.go | 51 --- spvwallet/log.go | 28 -- spvwallet/rpc/client.go | 73 ---- spvwallet/rpc/fucntions.go | 42 -- spvwallet/rpc/protocol.go | 46 -- spvwallet/rpc/server.go | 78 ---- spvwallet/sutil/tx.go | 40 -- spvwallet/wallet.go | 287 ------------- sync/config.go | 7 +- sync/manager.go | 18 +- util/block.go | 4 +- util/header.go | 19 +- util/interface.go | 23 + util/outpoint.go | 54 +++ util/tx.go | 63 ++- wallet/client.go | 37 ++ .../client/account/account.go | 8 +- {spvwallet => wallet}/client/common.go | 6 +- .../client/database/database.go | 34 +- wallet/client/database/interface.go | 18 + {spvwallet => wallet}/client/keystore.go | 53 +-- {spvwallet => wallet}/client/keystore_file.go | 0 .../client/transaction/transaction.go | 60 +-- .../interface.go => wallet/client/wallet.go | 149 +++---- {spvwallet => wallet}/client/wallet/wallet.go | 2 +- {spvwallet/rpc => wallet}/log.go | 2 +- {spvwallet => wallet}/store/headers/cache.go | 0 .../store/headers/database.go | 85 ++-- {spvwallet => wallet}/store/headers/log.go | 0 {spvwallet => wallet}/store/log.go | 4 +- {spvwallet => wallet}/store/sqlite/addrs.go | 2 +- .../store/sqlite/addrsbatch.go | 0 .../store/sqlite/database.go | 2 +- .../store/sqlite/databatch.go | 2 +- .../store/sqlite/interface.go | 26 +- {spvwallet => wallet}/store/sqlite/log.go | 0 {spvwallet => wallet}/store/sqlite/state.go | 0 {spvwallet => wallet}/store/sqlite/stxos.go | 18 +- .../store/sqlite/stxosbatch.go | 7 +- {spvwallet => wallet}/store/sqlite/txs.go | 41 +- .../store/sqlite/txsbatch.go | 13 +- {spvwallet => wallet}/store/sqlite/utxos.go | 19 +- .../store/sqlite/utxosbatch.go | 9 +- {spvwallet => wallet}/sutil/addr.go | 0 {spvwallet => wallet}/sutil/stxo.go | 0 {spvwallet => wallet}/sutil/utxo.go | 7 +- 76 files changed, 1880 insertions(+), 1235 deletions(-) rename blockchain/{chain.go => blockchain.go} (89%) create mode 100644 bloom/bloom.go create mode 100644 bloom/filter.go create mode 100644 bloom/merkleproof.go create mode 100644 bloom/murmurhash3.go create mode 100644 config.go rename blockchain/genesis.go => interface/blockheader.go (76%) delete mode 100644 sdk/bloom.go create mode 100644 spvwallet.go delete mode 100644 spvwallet/config/config.go delete mode 100644 spvwallet/log.go delete mode 100644 spvwallet/rpc/client.go delete mode 100644 spvwallet/rpc/fucntions.go delete mode 100644 spvwallet/rpc/protocol.go delete mode 100644 spvwallet/rpc/server.go delete mode 100644 spvwallet/sutil/tx.go delete mode 100644 spvwallet/wallet.go create mode 100644 util/interface.go create mode 100644 util/outpoint.go create mode 100644 wallet/client.go rename {spvwallet => wallet}/client/account/account.go (95%) rename {spvwallet => wallet}/client/common.go (96%) rename {spvwallet => wallet}/client/database/database.go (66%) create mode 100644 wallet/client/database/interface.go rename {spvwallet => wallet}/client/keystore.go (78%) rename {spvwallet => wallet}/client/keystore_file.go (100%) rename {spvwallet => wallet}/client/transaction/transaction.go (85%) rename spvwallet/client/interface.go => wallet/client/wallet.go (63%) rename {spvwallet => wallet}/client/wallet/wallet.go (98%) rename {spvwallet/rpc => wallet}/log.go (97%) rename {spvwallet => wallet}/store/headers/cache.go (100%) rename {spvwallet => wallet}/store/headers/database.go (60%) rename {spvwallet => wallet}/store/headers/log.go (100%) rename {spvwallet => wallet}/store/log.go (72%) rename {spvwallet => wallet}/store/sqlite/addrs.go (97%) rename {spvwallet => wallet}/store/sqlite/addrsbatch.go (100%) rename {spvwallet => wallet}/store/sqlite/database.go (98%) rename {spvwallet => wallet}/store/sqlite/databatch.go (99%) rename {spvwallet => wallet}/store/sqlite/interface.go (84%) rename {spvwallet => wallet}/store/sqlite/log.go (100%) rename {spvwallet => wallet}/store/sqlite/state.go (100%) rename {spvwallet => wallet}/store/sqlite/stxos.go (87%) rename {spvwallet => wallet}/store/sqlite/stxosbatch.go (84%) rename {spvwallet => wallet}/store/sqlite/txs.go (68%) rename {spvwallet => wallet}/store/sqlite/txsbatch.go (65%) rename {spvwallet => wallet}/store/sqlite/utxos.go (83%) rename {spvwallet => wallet}/store/sqlite/utxosbatch.go (74%) rename {spvwallet => wallet}/sutil/addr.go (100%) rename {spvwallet => wallet}/sutil/stxo.go (100%) rename {spvwallet => wallet}/sutil/utxo.go (93%) diff --git a/Makefile b/Makefile index 2948f28..81e6728 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ BUILD=go build VERSION := $(shell git describe --abbrev=4 --dirty --always --tags) -BUILD_SPV_CLI =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o ela-wallet client.go -BUILD_SPV_SERVICE =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o service log.go main.go +BUILD_SPV_CLI =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o ela-wallet log.go config.go client.go +BUILD_SPV_SERVICE =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o service log.go config.go spvwallet.go main.go all: $(BUILD_SPV_CLI) diff --git a/blockchain/chain.go b/blockchain/blockchain.go similarity index 89% rename from blockchain/chain.go rename to blockchain/blockchain.go index cc38193..d1d33ad 100644 --- a/blockchain/chain.go +++ b/blockchain/blockchain.go @@ -29,28 +29,17 @@ type BlockChain struct { } // NewBlockChain returns a new BlockChain instance. -func New(foundation string, db database.ChainStore) (*BlockChain, error) { - chain := &BlockChain{db: db} - +func New(genesisHeader util.BlockHeader, db database.ChainStore) (*BlockChain, error) { // Init genesis header - _, err := chain.db.Headers().GetBest() + _, err := db.Headers().GetBest() if err != nil { - var err error - var foundationAddress *common.Uint168 - if len(foundation) == 34 { - foundationAddress, err = common.Uint168FromAddress(foundation) - } else { - foundationAddress, err = common.Uint168FromAddress("8VYXVxKKSAxkmRrfmGpQR2Kc66XhG6m3ta") - } - if err != nil { - return nil, errors.New("parse foundation address failed") + storeHeader := &util.Header{BlockHeader: genesisHeader, TotalWork: new(big.Int)} + if err := db.Headers().Put(storeHeader, true); err != nil { + return nil, err } - genesisHeader := GenesisHeader(foundationAddress) - storeHeader := &util.Header{Header: *genesisHeader, TotalWork: new(big.Int)} - chain.db.Headers().Put(storeHeader, true) } - return chain, nil + return &BlockChain{db: db}, nil } func (b *BlockChain) CommitBlock(block *util.Block) (newTip, reorg bool, newHeight, fps uint32, err error) { @@ -70,7 +59,7 @@ func (b *BlockChain) CommitBlock(block *util.Block) (newTip, reorg bool, newHeig // If the tip is also the parent of this header, then we can save a database read by skipping // the lookup of the parent header. Otherwise (ophan?) we need to fetch the parent. - if block.Previous.IsEqual(tipHash) { + if hash := block.Previous(); hash.IsEqual(tipHash) { parentHeader = bestHeader } else { parentHeader, err = b.db.Headers().GetPrevious(header) @@ -88,7 +77,7 @@ func (b *BlockChain) CommitBlock(block *util.Block) (newTip, reorg bool, newHeig return false, false, 0, 0, nil } // Add the work of this header to the total work stored at the previous header - cumulativeWork := new(big.Int).Add(parentHeader.TotalWork, CalcWork(header.Bits)) + cumulativeWork := new(big.Int).Add(parentHeader.TotalWork, CalcWork(header.Bits())) // If the cumulative work is greater than the total work of our best header // then we have a new best header. Update the chain tip and check for a reorg. @@ -141,7 +130,7 @@ func (b *BlockChain) checkHeader(header *util.Header, prevHeader *util.Header) b height := prevHeader.Height // Check if headers link together. That whole 'blockchain' thing. - if prevHash.IsEqual(header.Previous) == false { + if prevHash.IsEqual(header.Previous()) == false { log.Errorf("Headers %d and %d don't link.\n", height, height+1) return false } diff --git a/blockchain/difficulty.go b/blockchain/difficulty.go index 3b748e3..0e7ade8 100644 --- a/blockchain/difficulty.go +++ b/blockchain/difficulty.go @@ -25,7 +25,7 @@ func CalcWork(bits uint32) *big.Int { func checkProofOfWork(header util.Header) bool { // The target difficulty must be larger than zero. - target := CompactToBig(header.Bits) + target := CompactToBig(header.Bits()) if target.Sign() <= 0 { return false } @@ -36,7 +36,7 @@ func checkProofOfWork(header util.Header) bool { } // The block hash must be less than the claimed target. - hash := header.AuxPow.ParBlockHeader.Hash() + hash := header.PowHash() hashNum := HashToBig(&hash) if hashNum.Cmp(target) > 0 { return false diff --git a/bloom/bloom.go b/bloom/bloom.go new file mode 100644 index 0000000..5d6f1ff --- /dev/null +++ b/bloom/bloom.go @@ -0,0 +1,206 @@ +package bloom + +import ( + "fmt" + "github.com/elastos/Elastos.ELA.SPV/fprate" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" +) + +// Build a bloom filter by giving the interested addresses and outpoints +func BuildBloomFilter(addresses []*common.Uint168, outpoints []*util.OutPoint) *Filter { + elements := uint32(len(addresses) + len(outpoints)) + + filter := NewFilter(elements, 0, fprate.ReducedFalsePositiveRate) + for _, address := range addresses { + filter.Add(address.Bytes()) + } + + for _, op := range outpoints { + filter.Add(op.Bytes()) + } + + return filter +} + +// Add a address into the given bloom filter +func FilterAddress(filter Filter, address *common.Uint168) { + filter.Add(address.Bytes()) +} + +// Add a outpoint into the given bloom filter +func FilterOutpoint(filter Filter, op *util.OutPoint) { + filter.Add(op.Bytes()) +} + +type merkleNode struct { + p uint32 // position in the binary tree + h *common.Uint256 // hash +} + +func (node merkleNode) String() string { + return fmt.Sprint("Node{pos:", node.p, ", hash:", node.h, "}") +} + +// given n merkle leaves, how deep is the tree? +// iterate shifting left until greater than n +func treeDepth(n uint32) (e uint32) { + for ; (1 << e) < n; e++ { + } + return +} + +// smallest power of 2 that can contain n +func nextPowerOfTwo(n uint32) uint32 { + return 1 << treeDepth(n) // 2^exponent +} + +// check if a node is populated based on node position and size of tree +func inDeadZone(pos, size uint32) bool { + msb := nextPowerOfTwo(size) + last := size - 1 // last valid position is 1 less than size + if pos > (msb<<1)-2 { // greater than root; not even in the tree + return true + } + h := msb + for pos >= h { + h = h>>1 | msb + last = last>>1 | msb + } + return pos > last +} + +func merkleParent(left *common.Uint256, right *common.Uint256) (*common.Uint256, error) { + // dupes can screw things up; CVE-2012-2459. check for them + if left != nil && right != nil && left.IsEqual(*right) { + return nil, fmt.Errorf("DUP HASH CRASH") + } + // if left child is nil, output nil. Need this for hard mode. + if left == nil { + return nil, fmt.Errorf("Left child is nil") + } + // if right is nil, hash left with itself + if right == nil { + right = left + } + + // Concatenate the left and right nodes + var sha [64]byte + copy(sha[:32], left[:]) + copy(sha[32:], right[:]) + + parent := common.Uint256(common.Sha256D(sha[:])) + return &parent, nil +} + +// take in a merkle block, parse through it, and return txids indicated +// If there's any problem return an error. Checks self-consistency only. +// doing it with a stack instead of recursion. Because... +// OK I don't know why I'm just not in to recursion OK? +func CheckMerkleBlock(m msg.MerkleBlock) ([]*common.Uint256, error) { + if m.Transactions == 0 { + return nil, fmt.Errorf("No transactions in merkleblock") + } + if len(m.Flags) == 0 { + return nil, fmt.Errorf("No flag bits") + } + var header = m.Header.(util.BlockHeader) + var s []merkleNode // the stack + var r []*common.Uint256 // slice to return; txids we care about + + // set initial position to root of merkle tree + msb := nextPowerOfTwo(m.Transactions) // most significant bit possible + pos := (msb << 1) - 2 // current position in tree + + var i uint8 // position in the current flag byte + var tip int + // main loop + for { + tip = len(s) - 1 // slice position of stack tip + // First check if stack operations can be performed + // is stack one filled item? that's complete. + if tip == 0 && s[0].h != nil { + if s[0].h.IsEqual(header.MerkleRoot()) { + return r, nil + } + return nil, fmt.Errorf("computed root %s but expect %s\n", + s[0].h.String(), header.MerkleRoot().String()) + } + // is current position in the tree's dead zone? partial parent + if inDeadZone(pos, m.Transactions) { + // create merkle parent from single side (left) + h, err := merkleParent(s[tip].h, nil) + if err != nil { + return r, err + } + s[tip-1].h = h + s = s[:tip] // remove 1 from stack + pos = s[tip-1].p | 1 // move position to parent's sibling + continue + } + // does stack have 3+ items? and are last 2 items filled? + if tip > 1 && s[tip-1].h != nil && s[tip].h != nil { + //fmt.Printf("nodes %d and %d combine into %d\n", + // s[tip-1].p, s[tip].p, s[tip-2].p) + // combine two filled nodes into parent node + h, err := merkleParent(s[tip-1].h, s[tip].h) + if err != nil { + return r, err + } + s[tip-2].h = h + // remove children + s = s[:tip-1] + // move position to parent's sibling + pos = s[tip-2].p | 1 + continue + } + + // no stack ops to perform, so make new node from message hashes + if len(m.Hashes) == 0 { + return nil, fmt.Errorf("Ran out of hashes at position %d.", pos) + } + if len(m.Flags) == 0 { + return nil, fmt.Errorf("Ran out of flag bits.") + } + var n merkleNode // make new node + n.p = pos // set current position for new node + + if pos&msb != 0 { // upper non-txid hash + if m.Flags[0]&(1<>1 | msb + } else { // left side, go to sibling + pos |= 1 + } + } else { // flag bit says skip; put empty on stack and descend + pos = (pos ^ msb) << 1 // descend to left + } + s = append(s, n) // push new node on stack + } else { // bottom row txid; flag bit indicates tx of interest + if pos >= m.Transactions { + // this can't happen because we check deadzone above... + return nil, fmt.Errorf("got into an invalid txid node") + } + n.h = m.Hashes[0] // copy hash from message + m.Hashes = m.Hashes[1:] // pop off message + if m.Flags[0]&(1< 1.0 { + fprate = 1.0 + } + if fprate < 1e-9 { + fprate = 1e-9 + } + + // Calculate the size of the filter in bytes for the given number of + // elements and false positive rate. + // + // Equivalent to m = -(n*ln(p) / ln(2)^2), where m is in bits. + // Then clamp it to the maximum filter size and convert to bytes. + dataLen := uint32(-1 * float64(elements) * math.Log(fprate) / ln2Squared) + dataLen = minUint32(dataLen, MaxFilterLoadFilterSize*8) / 8 + + // Calculate the number of hash functions based on the size of the + // filter calculated above and the number of elements. + // + // Equivalent to k = (m/n) * ln(2) + // Then clamp it to the maximum allowed hash funcs. + hashFuncs := uint32(float64(dataLen*8) / float64(elements) * math.Ln2) + hashFuncs = minUint32(hashFuncs, MaxFilterLoadHashFuncs) + + msg := &msg.FilterLoad{ + Filter: make([]byte, dataLen), + HashFuncs: hashFuncs, + Tweak: tweak, + } + return &Filter{msg: msg} +} + +// LoadFilter creates a new Filter instance with the given underlying +// msg.FilterLoad. +func LoadFilter(msg *msg.FilterLoad) *Filter { + filter := new(Filter) + filter.msg = msg + return filter +} + +// IsLoaded returns true if a filter is loaded, otherwise false. +// +// This function is safe for concurrent access. +func (bf *Filter) IsLoaded() bool { + bf.mtx.Lock() + loaded := bf.msg != nil + bf.mtx.Unlock() + return loaded +} + +// Reload loads a new filter replacing any existing filter. +// +// This function is safe for concurrent access. +func (bf *Filter) Reload(msg *msg.FilterLoad) { + bf.mtx.Lock() + bf.msg = msg + bf.mtx.Unlock() +} + +// Unload unloads the bloom filter. +// +// This function is safe for concurrent access. +func (bf *Filter) Unload() { + bf.mtx.Lock() + bf.msg = nil + bf.mtx.Unlock() +} + +// hash returns the bit offset in the bloom filter which corresponds to the +// passed data for the given indepedent hash function number. +func (bf *Filter) hash(hashNum uint32, data []byte) uint32 { + // bitcoind: 0xfba4c795 chosen as it guarantees a reasonable bit + // difference between hashNum values. + // + // Note that << 3 is equivalent to multiplying by 8, but is faster. + // Thus the returned hash is brought into range of the number of bits + // the filter has and returned. + mm := MurmurHash3(hashNum*0xfba4c795+bf.msg.Tweak, data) + return mm % (uint32(len(bf.msg.Filter)) << 3) +} + +// matches returns true if the bloom filter might contain the passed data and +// false if it definitely does not. +// +// This function MUST be called with the filter lock held. +func (bf *Filter) matches(data []byte) bool { + if bf.msg == nil { + return false + } + + // The bloom filter does not contain the data if any of the bit offsets + // which result from hashing the data using each independent hash + // function are not set. The shifts and masks below are a faster + // equivalent of: + // arrayIndex := idx / 8 (idx >> 3) + // bitOffset := idx % 8 (idx & 7) + /// if filter[arrayIndex] & 1<>3]&(1<<(idx&7)) == 0 { + return false + } + } + return true +} + +// Matches returns true if the bloom filter might contain the passed data and +// false if it definitely does not. +// +// This function is safe for concurrent access. +func (bf *Filter) Matches(data []byte) bool { + bf.mtx.Lock() + match := bf.matches(data) + bf.mtx.Unlock() + return match +} + +// matchesOutPoint returns true if the bloom filter might contain the passed +// outpoint and false if it definitely does not. +// +// This function MUST be called with the filter lock held. +func (bf *Filter) matchesOutPoint(outpoint *core.OutPoint) bool { + return bf.matches(outpoint.Bytes()) +} + +// MatchesOutPoint returns true if the bloom filter might contain the passed +// outpoint and false if it definitely does not. +// +// This function is safe for concurrent access. +func (bf *Filter) MatchesOutPoint(outpoint *core.OutPoint) bool { + bf.mtx.Lock() + match := bf.matchesOutPoint(outpoint) + bf.mtx.Unlock() + return match +} + +// add adds the passed byte slice to the bloom filter. +// +// This function MUST be called with the filter lock held. +func (bf *Filter) add(data []byte) { + if bf.msg == nil { + return + } + + // Adding data to a bloom filter consists of setting all of the bit + // offsets which result from hashing the data using each independent + // hash function. The shifts and masks below are a faster equivalent + // of: + // arrayIndex := idx / 8 (idx >> 3) + // bitOffset := idx % 8 (idx & 7) + /// filter[arrayIndex] |= 1<>3] |= (1 << (7 & idx)) + } +} + +// Add adds the passed byte slice to the bloom filter. +// +// This function is safe for concurrent access. +func (bf *Filter) Add(data []byte) { + bf.mtx.Lock() + bf.add(data) + bf.mtx.Unlock() +} + +// AddHash adds the passed chainhash.Hash to the Filter. +// +// This function is safe for concurrent access. +func (bf *Filter) AddHash(hash *common.Uint256) { + bf.mtx.Lock() + bf.add(hash[:]) + bf.mtx.Unlock() +} + +// addOutPoint adds the passed tx outpoint to the bloom filter. +// +// This function MUST be called with the filter lock held. +func (bf *Filter) addOutPoint(outpoint *core.OutPoint) { + bf.add(outpoint.Bytes()) +} + +// AddOutPoint adds the passed tx outpoint to the bloom filter. +// +// This function is safe for concurrent access. +func (bf *Filter) AddOutPoint(outpoint *core.OutPoint) { + bf.mtx.Lock() + bf.addOutPoint(outpoint) + bf.mtx.Unlock() +} + +// matchTxAndUpdate returns true if the bloom filter matches data within the +// passed tx, otherwise false is returned. If the filter does match +// the passed tx, it will also update the filter depending on the bloom +// update flags set via the loaded filter if needed. +// +// This function MUST be called with the filter lock held. +func (bf *Filter) matchTxAndUpdate(txn *core.Transaction) bool { + // Check if the filter matches the hash of the tx. + // This is useful for finding transactions when they appear in a block. + hash := txn.Hash() + matched := bf.matches(hash[:]) + + for i, txOut := range txn.Outputs { + if !bf.matches(txOut.ProgramHash[:]) { + continue + } + + matched = true + bf.addOutPoint(core.NewOutPoint(txn.Hash(), uint16(i))) + } + + // Nothing more to do if a match has already been made. + if matched { + return true + } + + // At this point, the tx and none of the data elements in the + // public key scripts of its outputs matched. + + // Check if the filter matches any outpoints this tx spends + for _, txIn := range txn.Inputs { + if bf.matchesOutPoint(&txIn.Previous) { + return true + } + } + + return false +} + +// MatchTxAndUpdate returns true if the bloom filter matches data within the +// passed tx, otherwise false is returned. If the filter does match +// the passed tx, it will also update the filter depending on the bloom +// update flags set via the loaded filter if needed. +// +// This function is safe for concurrent access. +func (bf *Filter) MatchTxAndUpdate(tx *core.Transaction) bool { + bf.mtx.Lock() + match := bf.matchTxAndUpdate(tx) + bf.mtx.Unlock() + return match +} + +func (bf *Filter) GetFilterLoadMsg() *msg.FilterLoad { + return bf.msg +} diff --git a/bloom/merkleproof.go b/bloom/merkleproof.go new file mode 100644 index 0000000..75e9605 --- /dev/null +++ b/bloom/merkleproof.go @@ -0,0 +1,85 @@ +package bloom + +import ( + "fmt" + "io" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" +) + +// maxFlagsPerMerkleProof is the maximum number of flag bytes that could +// possibly fit into a merkle proof. Since each transaction is represented by +// a single bit, this is the max number of transactions per block divided by +// 8 bits per byte. Then an extra one to cover partials. +const maxFlagsPerMerkleProof = msg.MaxTxPerBlock / 8 + +type MerkleProof struct { + BlockHash common.Uint256 + Height uint32 + Transactions uint32 + Hashes []*common.Uint256 + Flags []byte +} + +func (p *MerkleProof) Serialize(w io.Writer) error { + // Read num transaction hashes and limit to max. + numHashes := len(p.Hashes) + if numHashes > msg.MaxTxPerBlock { + str := fmt.Sprintf("too many transaction hashes for message "+ + "[count %v, max %v]", numHashes, msg.MaxTxPerBlock) + return common.FuncError("MerkleProof.Serialize", str) + } + numFlagBytes := len(p.Flags) + if numFlagBytes > maxFlagsPerMerkleProof { + str := fmt.Sprintf("too many flag bytes for message [count %v, "+ + "max %v]", numFlagBytes, maxFlagsPerMerkleProof) + return common.FuncError("MerkleProof.Serialize", str) + } + + err := common.WriteElements(w, &p.BlockHash, p.Height, p.Transactions, + uint32(numHashes)) + if err != nil { + return err + } + + for _, hash := range p.Hashes { + if err := hash.Serialize(w); err != nil { + return err + } + } + + return common.WriteVarBytes(w, p.Flags) +} + +func (p *MerkleProof) Deserialize(r io.Reader) error { + var numHashes uint32 + err := common.ReadElements(r, + &p.BlockHash, + &p.Height, + &p.Transactions, + &numHashes, + ) + if err != nil { + return err + } + + if numHashes > msg.MaxTxPerBlock { + return fmt.Errorf("MerkleProof.Deserialize too many transaction"+ + " hashes for message [count %v, max %v]", numHashes, msg.MaxTxPerBlock) + } + + hashes := make([]common.Uint256, numHashes) + p.Hashes = make([]*common.Uint256, 0, numHashes) + for i := uint32(0); i < numHashes; i++ { + hash := &hashes[i] + if err := hash.Deserialize(r); err != nil { + return err + } + p.Hashes = append(p.Hashes, hash) + } + + p.Flags, err = common.ReadVarBytes(r, maxFlagsPerMerkleProof, + "merkle proof flags size") + return err +} diff --git a/bloom/murmurhash3.go b/bloom/murmurhash3.go new file mode 100644 index 0000000..6bb562e --- /dev/null +++ b/bloom/murmurhash3.go @@ -0,0 +1,72 @@ +// Copyright (c) 2013, 2014 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package bloom + +import ( + "encoding/binary" +) + +// The following constants are used by the MurmurHash3 algorithm. +const ( + murmurC1 = 0xcc9e2d51 + murmurC2 = 0x1b873593 + murmurR1 = 15 + murmurR2 = 13 + murmurM = 5 + murmurN = 0xe6546b64 +) + +// MurmurHash3 implements a non-cryptographic hash function using the +// MurmurHash3 algorithm. This implementation yields a 32-bit hash value which +// is suitable for general hash-based lookups. The seed can be used to +// effectively randomize the hash function. This makes it ideal for use in +// bloom filters which need multiple independent hash functions. +func MurmurHash3(seed uint32, data []byte) uint32 { + dataLen := uint32(len(data)) + hash := seed + k := uint32(0) + numBlocks := dataLen / 4 + + // Calculate the hash in 4-byte chunks. + for i := uint32(0); i < numBlocks; i++ { + k = binary.LittleEndian.Uint32(data[i*4:]) + k *= murmurC1 + k = (k << murmurR1) | (k >> (32 - murmurR1)) + k *= murmurC2 + + hash ^= k + hash = (hash << murmurR2) | (hash >> (32 - murmurR2)) + hash = hash*murmurM + murmurN + } + + // Handle remaining bytes. + tailIdx := numBlocks * 4 + k = 0 + + switch dataLen & 3 { + case 3: + k ^= uint32(data[tailIdx+2]) << 16 + fallthrough + case 2: + k ^= uint32(data[tailIdx+1]) << 8 + fallthrough + case 1: + k ^= uint32(data[tailIdx]) + k *= murmurC1 + k = (k << murmurR1) | (k >> (32 - murmurR1)) + k *= murmurC2 + hash ^= k + } + + // Finalization. + hash ^= dataLen + hash ^= hash >> 16 + hash *= 0x85ebca6b + hash ^= hash >> 13 + hash *= 0xc2b2ae35 + hash ^= hash >> 16 + + return hash +} diff --git a/client.go b/client.go index 27a7f55..5ce265d 100644 --- a/client.go +++ b/client.go @@ -1,34 +1,38 @@ package main import ( - "os" + "fmt" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/client/account" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/client/transaction" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/client/wallet" + "github.com/elastos/Elastos.ELA.SPV/wallet" - "github.com/urfave/cli" + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" ) var Version string func main() { - app := cli.NewApp() - app.Name = "ELASTOS SPV WALLET" - app.Version = Version - app.HelpName = "ELASTOS SPV WALLET HELP" - app.Usage = "command line user interface" - app.UsageText = "[global option] command [command options] [args]" - app.HideHelp = false - app.HideVersion = false - //commands - app.Commands = []cli.Command{ - wallet.NewCreateCommand(), - wallet.NewChangePasswordCommand(), - wallet.NewResetCommand(), - account.NewCommand(), - transaction.NewCommand(), - } + url := fmt.Sprint("http://127.0.0.1:", config.JsonRpcPort, "/spvwallet/") + wallet.RunClient(Version, url, getSystemAssetId(), newBlockHeader) +} - app.Run(os.Args) +func getSystemAssetId() common.Uint256 { + systemToken := &core.Transaction{ + TxType: core.RegisterAsset, + PayloadVersion: 0, + Payload: &core.PayloadRegisterAsset{ + Asset: core.Asset{ + Name: "ELA", + Precision: 0x08, + AssetType: 0x00, + }, + Amount: 0 * 100000000, + Controller: common.Uint168{}, + }, + Attributes: []*core.Attribute{}, + Inputs: []*core.Input{}, + Outputs: []*core.Output{}, + Programs: []*core.Program{}, + } + return systemToken.Hash() } diff --git a/config.go b/config.go new file mode 100644 index 0000000..d06aaff --- /dev/null +++ b/config.go @@ -0,0 +1,84 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" + "io/ioutil" + "os" +) + +const ( + ConfigFilename = "./config.json" +) + +var config = loadConfig() + +type Config struct { + Magic uint32 + SeedList []string + NodePort uint16 // node port for public peers to provide services. + Foundation string + PrintLevel int + MaxLogsSize int64 + MaxPerLogSize int64 + JsonRpcPort uint16 + + foundation *common.Uint168 +} + +func loadConfig() *Config { + data, err := ioutil.ReadFile(ConfigFilename) + if err != nil { + fmt.Printf("Read config file error %s", err) + os.Exit(-1) + } + // Remove the UTF-8 Byte Order Mark + data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf")) + + c := Config{} + err = json.Unmarshal(data, &c) + if err != nil { + fmt.Printf("Read config file error %s", err) + os.Exit(-1) + } + + if c.Foundation == "" { + c.Foundation = "8VYXVxKKSAxkmRrfmGpQR2Kc66XhG6m3ta" + } + + c.foundation, err = common.Uint168FromAddress(c.Foundation) + if err != nil { + fmt.Printf("Parse foundation address error %s", err) + os.Exit(-1) + } + + return &c +} + +type blockHeader struct { + *core.Header +} + +func (h *blockHeader) Previous() common.Uint256 { + return h.Header.Previous +} + +func (h *blockHeader) Bits() uint32 { + return h.Header.Bits +} + +func (h *blockHeader) MerkleRoot() common.Uint256 { + return h.Header.MerkleRoot +} + +func (h *blockHeader) PowHash() common.Uint256 { + return h.AuxPow.ParBlockHeader.Hash() +} + +func newBlockHeader() util.BlockHeader { + return &blockHeader{Header: &core.Header{}} +} diff --git a/database/defaultdb.go b/database/defaultdb.go index c937a88..a740f0c 100644 --- a/database/defaultdb.go +++ b/database/defaultdb.go @@ -28,7 +28,7 @@ func (d *defaultChainDB) CommitBlock(block *util.Block, newTip bool) (fps uint32 batch := d.t.Batch() for _, tx := range block.Transactions { - fp, err := batch.PutTx(util.NewTx(*tx, block.Height)) + fp, err := batch.PutTx(tx, block.Height) if err != nil { return 0, batch.Rollback() } diff --git a/database/txsdb.go b/database/txsdb.go index 6663acc..aa9b55a 100644 --- a/database/txsdb.go +++ b/database/txsdb.go @@ -29,7 +29,7 @@ type TxsDB interface { type TxBatch interface { // PutTx add a store transaction operation into batch, and return // if it is a false positive and error. - PutTx(tx *util.Tx) (bool, error) + PutTx(tx util.Transaction, height uint32) (bool, error) // DelTx add a delete transaction operation into batch. DelTx(txId *common.Uint256) error diff --git a/blockchain/genesis.go b/interface/blockheader.go similarity index 76% rename from blockchain/genesis.go rename to interface/blockheader.go index 9407e01..ff470d0 100644 --- a/blockchain/genesis.go +++ b/interface/blockheader.go @@ -1,16 +1,42 @@ -package blockchain +package _interface import ( "time" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" "github.com/elastos/Elastos.ELA/core" ) +type blockHeader struct { + *core.Header +} + +func (h *blockHeader) Previous() common.Uint256 { + return h.Header.Previous +} + +func (h *blockHeader) Bits() uint32 { + return h.Header.Bits +} + +func (h *blockHeader) MerkleRoot() common.Uint256 { + return h.Header.MerkleRoot +} + +func (h *blockHeader) PowHash() common.Uint256 { + return h.AuxPow.ParBlockHeader.Hash() +} + +func newBlockHeader() util.BlockHeader { + return &blockHeader{Header: &core.Header{}} +} + // GenesisHeader creates a specific genesis header by the given // foundation address. -func GenesisHeader(foundation *common.Uint168) *core.Header { +func GenesisHeader(foundation *common.Uint168) util.BlockHeader { // Genesis time genesisTime := time.Date(2017, time.December, 22, 10, 0, 0, 0, time.UTC) @@ -81,5 +107,5 @@ func GenesisHeader(foundation *common.Uint168) *core.Header { } header.MerkleRoot, _ = crypto.ComputeRoot(hashes) - return &header + return &blockHeader{Header: &header} } diff --git a/interface/interface.go b/interface/interface.go index 1419bab..e59c3c3 100644 --- a/interface/interface.go +++ b/interface/interface.go @@ -1,10 +1,10 @@ package _interface import ( + "github.com/elastos/Elastos.ELA.SPV/bloom" "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/bloom" "github.com/elastos/Elastos.ELA/core" ) @@ -20,7 +20,7 @@ type Config struct { // The public seed peers addresses. SeedList []string - // DefaultPort is the default port for public peers provide services. + // NodePort is the default port for public peers provide services. DefaultPort uint16 // The min candidate peers count to start syncing progress. diff --git a/interface/interface_test.go b/interface/interface_test.go index d5b4a7e..5c65dc9 100644 --- a/interface/interface_test.go +++ b/interface/interface_test.go @@ -2,21 +2,23 @@ package _interface import ( "fmt" + "os" "testing" "time" "github.com/elastos/Elastos.ELA.SPV/blockchain" - spvpeer "github.com/elastos/Elastos.ELA.SPV/peer" + "github.com/elastos/Elastos.ELA.SPV/bloom" + "github.com/elastos/Elastos.ELA.SPV/peer" "github.com/elastos/Elastos.ELA.SPV/sdk" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" "github.com/elastos/Elastos.ELA.SPV/sync" + "github.com/elastos/Elastos.ELA.SPV/wallet/store" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/elalog" + "github.com/elastos/Elastos.ELA.Utility/http/jsonrpc" "github.com/elastos/Elastos.ELA.Utility/p2p/addrmgr" "github.com/elastos/Elastos.ELA.Utility/p2p/connmgr" "github.com/elastos/Elastos.ELA.Utility/p2p/server" - "github.com/elastos/Elastos.ELA/bloom" "github.com/elastos/Elastos.ELA/core" ) @@ -101,22 +103,41 @@ func TestGetListenerKey(t *testing.T) { } func TestNewSPVService(t *testing.T) { - addrmgr.UseLogger(elalog.Stdout) - connmgr.UseLogger(elalog.Stdout) - sdk.UseLogger(elalog.Stdout) - //rpc.UseLogger(logger) - //peer.UseLogger(elalog.Stdout) - spvpeer.UseLogger(elalog.Stdout) - server.UseLogger(elalog.Stdout) - blockchain.UseLogger(elalog.Stdout) - sync.UseLogger(elalog.Stdout) - UseLogger(elalog.Stdout) + backend := elalog.NewBackend(os.Stdout, elalog.Lshortfile) + admrlog := backend.Logger("ADMR", elalog.LevelOff) + cmgrlog := backend.Logger("CMGR", elalog.LevelOff) + bcdblog := backend.Logger("BCDB", elalog.LevelTrace) + synclog := backend.Logger("SYNC", elalog.LevelTrace) + peerlog := backend.Logger("PEER", elalog.LevelTrace) + spvslog := backend.Logger("SPVS", elalog.LevelTrace) + srvrlog := backend.Logger("SRVR", elalog.LevelTrace) + rpcslog := backend.Logger("RPCS", elalog.LevelTrace) + + addrmgr.UseLogger(admrlog) + connmgr.UseLogger(cmgrlog) + blockchain.UseLogger(bcdblog) + sdk.UseLogger(spvslog) + jsonrpc.UseLogger(rpcslog) + peer.UseLogger(peerlog) + server.UseLogger(srvrlog) + store.UseLogger(bcdblog) + sync.UseLogger(synclog) + + seedList := []string{ + "node-regtest-201.elastos.org:22866", + "node-regtest-202.elastos.org:22866", + "node-regtest-203.elastos.org:22866", + "node-regtest-204.elastos.org:22866", + "node-regtest-205.elastos.org:22866", + "node-regtest-206.elastos.org:22866", + "node-regtest-207.elastos.org:22866", + } config := &Config{ - Magic: config.Values().Magic, - Foundation: config.Values().Foundation, - SeedList: config.Values().SeedList, - DefaultPort: config.Values().DefaultPort, + Magic: 20180627, + Foundation: "8ZNizBf4KhhPjeJRGpox6rPcHE5Np6tFx3", + SeedList: seedList, + DefaultPort: 22866, MinOutbound: 8, MaxConnections: 100, } diff --git a/interface/keystoreimpl.go b/interface/keystoreimpl.go index 77ed67b..ea3824c 100644 --- a/interface/keystoreimpl.go +++ b/interface/keystoreimpl.go @@ -1,11 +1,11 @@ package _interface import ( - "github.com/elastos/Elastos.ELA.SPV/spvwallet/client" + "github.com/elastos/Elastos.ELA.SPV/wallet/client" ) type keystore struct { - keystore client.Keystore + keystore *client.Keystore } // This method will open or create a keystore with the given password @@ -51,6 +51,6 @@ func (impl *keystore) Json() (string, error) { } func (impl *keystore) FromJson(str string, password string) error { - impl.keystore = new(client.KeystoreImpl) + impl.keystore = new(client.Keystore) return impl.keystore.FromJson(str, password) } diff --git a/interface/spvservice.go b/interface/spvservice.go index c95986f..891325a 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "github.com/elastos/Elastos.ELA.SPV/bloom" "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/interface/store" "github.com/elastos/Elastos.ELA.SPV/sdk" @@ -13,7 +14,6 @@ import ( "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA/bloom" "github.com/elastos/Elastos.ELA/core" ) @@ -26,7 +26,16 @@ type spvservice struct { } func newSpvService(cfg *Config) (*spvservice, error) { - headerStore, err := store.NewHeaderStore() + if cfg.Foundation == "" { + cfg.Foundation = "8VYXVxKKSAxkmRrfmGpQR2Kc66XhG6m3ta" + } + + foundation, err := common.Uint168FromAddress(cfg.Foundation) + if err != nil { + return nil, fmt.Errorf("Parse foundation address error %s", err) + } + + headerStore, err := store.NewHeaderStore(newBlockHeader) if err != nil { return nil, err } @@ -51,10 +60,14 @@ func newSpvService(cfg *Config) (*spvservice, error) { DefaultPort: cfg.DefaultPort, MaxPeers: cfg.MaxConnections, MinPeersForSync: cfg.MinPeersForSync, - Foundation: cfg.Foundation, + GenesisHeader: GenesisHeader(foundation), ChainStore: chainStore, - GetFilterData: service.GetFilterData, - StateNotifier: service, + NewTransaction: func() util.Transaction { + return &core.Transaction{} + }, + NewBlockHeader: newBlockHeader, + GetFilterData: service.GetFilterData, + StateNotifier: service, } service.IService, err = sdk.NewService(serviceCfg) @@ -92,7 +105,7 @@ func (s *spvservice) VerifyTransaction(proof bloom.MerkleProof, tx core.Transact // Check if merkleroot is match merkleBlock := msg.MerkleBlock{ - Header: &header.Header, + Header: header.BlockHeader, Transactions: proof.Transactions, Hashes: proof.Hashes, Flags: proof.Flags, @@ -121,14 +134,14 @@ func (s *spvservice) VerifyTransaction(proof bloom.MerkleProof, tx core.Transact } func (s *spvservice) SendTransaction(tx core.Transaction) error { - return s.IService.SendTransaction(tx) + return s.IService.SendTransaction(&tx) } func (s *spvservice) HeaderStore() database.Headers { return s.headers } -func (s *spvservice) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { +func (s *spvservice) GetFilterData() ([]*common.Uint168, []*util.OutPoint) { ops, err := s.db.Ops().GetAll() if err != nil { log.Error("[SPV_SERVICE] GetData error ", err) @@ -171,16 +184,16 @@ func (s *spvservice) RemoveTxs(height uint32) (int, error) { } // TransactionAnnounce will be invoked when received a new announced transaction. -func (s *spvservice) TransactionAnnounce(tx *core.Transaction) {} +func (s *spvservice) TransactionAnnounce(tx util.Transaction) {} // TransactionAccepted will be invoked after a transaction sent by // SendTransaction() method has been accepted. Notice: this method needs at // lest two connected peers to work. -func (s *spvservice) TransactionAccepted(tx *core.Transaction) {} +func (s *spvservice) TransactionAccepted(tx util.Transaction) {} // TransactionRejected will be invoked if a transaction sent by SendTransaction() // method has been rejected. -func (s *spvservice) TransactionRejected(tx *core.Transaction) {} +func (s *spvservice) TransactionRejected(tx util.Transaction) {} // TransactionConfirmed will be invoked after a transaction sent by // SendTransaction() method has been packed into a block. @@ -192,7 +205,7 @@ func (s *spvservice) BlockCommitted(block *util.Block) { log.Infof("Receive block %s height %d", block.Hash(), block.Height) for _, tx := range block.Transactions { for _, listener := range s.listeners { - s.queueMessageByListener(listener, tx, block.Height) + s.queueMessageByListener(listener, tx.(*core.Transaction), block.Height) } } @@ -210,12 +223,18 @@ func (s *spvservice) BlockCommitted(block *util.Block) { } // Get transaction from db - storeTx, err := s.db.Txs().Get(&item.TxId) + utx, err := s.db.Txs().Get(&item.TxId) if err != nil { log.Errorf("query transaction failed, txId %s", item.TxId.String()) continue } + var tx core.Transaction + err = tx.Deserialize(bytes.NewReader(utx.RawData)) + if err != nil { + continue + } + // Notify listeners s.notifyTransaction( item.NotifyId, @@ -226,7 +245,7 @@ func (s *spvservice) BlockCommitted(block *util.Block) { Hashes: block.Hashes, Flags: block.Flags, }, - storeTx.Transaction, + tx, block.Height-item.Height, ) } @@ -331,19 +350,22 @@ type txBatch struct { // PutTx add a store transaction operation into batch, and return // if it is a false positive and error. -func (b *txBatch) PutTx(tx *util.Tx) (bool, error) { +func (b *txBatch) PutTx(utx util.Transaction, height uint32) (bool, error) { + tx := utx.(*core.Transaction) hits := make(map[common.Uint168]struct{}) - ops := make(map[*core.OutPoint]common.Uint168) + ops := make(map[*util.OutPoint]common.Uint168) for index, output := range tx.Outputs { if b.db.Addrs().GetFilter().ContainAddr(output.ProgramHash) { - outpoint := core.NewOutPoint(tx.Hash(), uint16(index)) + outpoint := util.NewOutPoint(tx.Hash(), uint16(index)) ops[outpoint] = output.ProgramHash hits[output.ProgramHash] = struct{}{} } } for _, input := range tx.Inputs { - if addr := b.db.Ops().IsExist(&input.Previous); addr != nil { + op := input.Previous + addr := b.db.Ops().HaveOp(util.NewOutPoint(op.TxID, op.Index)) + if addr != nil { hits[*addr] = struct{}{} } } @@ -370,23 +392,29 @@ func (b *txBatch) PutTx(tx *util.Tx) (bool, error) { b.batch.Que().Put(&store.QueItem{ NotifyId: getListenerKey(listener), TxId: tx.Hash(), - Height: tx.Height, + Height: height, }) } } - return false, b.batch.Txs().Put(tx) + return false, b.batch.Txs().Put(util.NewTx(utx, height)) } // DelTx add a delete transaction operation into batch. func (b *txBatch) DelTx(txId *common.Uint256) error { - tx, err := b.db.Txs().Get(txId) + utx, err := b.db.Txs().Get(txId) + if err != nil { + return err + } + + var tx core.Transaction + err = tx.Deserialize(bytes.NewReader(utx.RawData)) if err != nil { return err } for index := range tx.Outputs { - outpoint := core.NewOutPoint(tx.Hash(), uint16(index)) + outpoint := util.NewOutPoint(utx.Hash, uint16(index)) b.batch.Ops().Del(outpoint) } diff --git a/interface/store/databatch.go b/interface/store/databatch.go index 81ea6ec..60f7e52 100644 --- a/interface/store/databatch.go +++ b/interface/store/databatch.go @@ -54,15 +54,21 @@ func (b *dataBatch) DelAll(height uint32) error { txsBucket := b.boltTx.Bucket(BKTTxs) opsBucket := b.boltTx.Bucket(BKTOps) for txId := range txMap { - var txn util.Tx + var utx util.Tx data := txsBucket.Get(txId.Bytes()) - err := txn.Deserialize(bytes.NewReader(data)) + err := utx.Deserialize(bytes.NewReader(data)) if err != nil { return err } - for index := range txn.Outputs { - outpoint := core.NewOutPoint(txn.Hash(), uint16(index)).Bytes() + var tx core.Transaction + err = tx.Deserialize(bytes.NewReader(utx.RawData)) + if err != nil { + return err + } + + for index := range tx.Outputs { + outpoint := core.NewOutPoint(utx.Hash, uint16(index)).Bytes() opsBucket.Delete(outpoint) } diff --git a/interface/store/headers.go b/interface/store/headers.go index 6f44efd..5e70149 100644 --- a/interface/store/headers.go +++ b/interface/store/headers.go @@ -28,10 +28,11 @@ var _ HeaderStore = (*headers)(nil) type headers struct { *sync.RWMutex *bolt.DB - cache *cache + cache *cache + newHeader func() util.BlockHeader } -func NewHeaderStore() (*headers, error) { +func NewHeaderStore(newHeader func() util.BlockHeader) (*headers, error) { db, err := bolt.Open("headers.bin", 0644, &bolt.Options{InitialMmapSize: 5000000}) if err != nil { return nil, err @@ -54,9 +55,10 @@ func NewHeaderStore() (*headers, error) { }) headers := &headers{ - RWMutex: new(sync.RWMutex), - DB: db, - cache: newCache(100), + RWMutex: new(sync.RWMutex), + DB: db, + cache: newCache(100), + newHeader: newHeader, } headers.initCache() @@ -117,10 +119,14 @@ func (h *headers) Put(header *util.Header, newTip bool) error { } func (h *headers) GetPrevious(header *util.Header) (*util.Header, error) { + if header.Height == 0 { + return nil, fmt.Errorf("no more previous header") + } if header.Height == 1 { return &util.Header{TotalWork: new(big.Int)}, nil } - return h.Get(&header.Previous) + hash := header.Previous() + return h.Get(&hash) } func (h *headers) Get(hash *common.Uint256) (header *util.Header, err error) { @@ -134,7 +140,7 @@ func (h *headers) Get(hash *common.Uint256) (header *util.Header, err error) { err = h.View(func(tx *bolt.Tx) error { - header, err = getHeader(tx, BKTHeaders, hash.Bytes()) + header, err = h.getHeader(tx, BKTHeaders, hash.Bytes()) if err != nil { return err } @@ -155,7 +161,7 @@ func (h *headers) GetBest() (header *util.Header, err error) { err = h.View(func(tx *bolt.Tx) error { - header, err = getHeader(tx, BKTChainTip, KEYChainTip) + header, err = h.getHeader(tx, BKTChainTip, KEYChainTip) if err != nil { return err } @@ -174,7 +180,7 @@ func (h *headers) GetByHeight(height uint32) (header *util.Header, err error) { var key [4]byte binary.LittleEndian.PutUint32(key[:], height) hashBytes := tx.Bucket(BKTHeightHash).Get(key[:]) - header, err = getHeader(tx, BKTHeaders, hashBytes) + header, err = h.getHeader(tx, BKTHeaders, hashBytes) if err != nil { return err } @@ -208,13 +214,14 @@ func (h *headers) Close() error { return h.DB.Close() } -func getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*util.Header, error) { +func (h *headers) getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*util.Header, error) { headerBytes := tx.Bucket(bucket).Get(key) if headerBytes == nil { return nil, fmt.Errorf("header %s does not exist in database", hex.EncodeToString(key)) } var header util.Header + header.BlockHeader = h.newHeader() err := header.Deserialize(headerBytes) if err != nil { return nil, err diff --git a/interface/store/interface.go b/interface/store/interface.go index 66f9d59..7e17bb3 100644 --- a/interface/store/interface.go +++ b/interface/store/interface.go @@ -6,7 +6,6 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" ) type HeaderStore interface { @@ -64,16 +63,16 @@ type TxsBatch interface { type Ops interface { database.DB - Put(*core.OutPoint, common.Uint168) error - IsExist(*core.OutPoint) *common.Uint168 - GetAll() ([]*core.OutPoint, error) + Put(*util.OutPoint, common.Uint168) error + HaveOp(*util.OutPoint) *common.Uint168 + GetAll() ([]*util.OutPoint, error) Batch() OpsBatch } type OpsBatch interface { batch - Put(*core.OutPoint, common.Uint168) error - Del(*core.OutPoint) error + Put(*util.OutPoint, common.Uint168) error + Del(*util.OutPoint) error } type Que interface { diff --git a/interface/store/ops.go b/interface/store/ops.go index ccf568b..e59eebe 100644 --- a/interface/store/ops.go +++ b/interface/store/ops.go @@ -3,9 +3,10 @@ package store import ( "sync" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/boltdb/bolt" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" ) var ( @@ -36,7 +37,7 @@ func NewOps(db *bolt.DB) (*ops, error) { return store, nil } -func (o *ops) Put(op *core.OutPoint, addr common.Uint168) (err error) { +func (o *ops) Put(op *util.OutPoint, addr common.Uint168) (err error) { o.Lock() defer o.Unlock() return o.Update(func(tx *bolt.Tx) error { @@ -44,7 +45,7 @@ func (o *ops) Put(op *core.OutPoint, addr common.Uint168) (err error) { }) } -func (o *ops) IsExist(op *core.OutPoint) (addr *common.Uint168) { +func (o *ops) HaveOp(op *util.OutPoint) (addr *common.Uint168) { o.RLock() defer o.RUnlock() @@ -59,13 +60,13 @@ func (o *ops) IsExist(op *core.OutPoint) (addr *common.Uint168) { return addr } -func (o *ops) GetAll() (ops []*core.OutPoint, err error) { +func (o *ops) GetAll() (ops []*util.OutPoint, err error) { o.RLock() defer o.RUnlock() err = o.View(func(tx *bolt.Tx) error { return tx.Bucket(BKTOps).ForEach(func(k, v []byte) error { - op, err := core.OutPointFromBytes(k) + op, err := util.OutPointFromBytes(k) if err != nil { return err } diff --git a/interface/store/opsbatch.go b/interface/store/opsbatch.go index d5c6892..c011474 100644 --- a/interface/store/opsbatch.go +++ b/interface/store/opsbatch.go @@ -3,9 +3,10 @@ package store import ( "sync" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/boltdb/bolt" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" ) // Ensure opsBatch implement OpsBatch interface. @@ -16,13 +17,13 @@ type opsBatch struct { *bolt.Tx } -func (b *opsBatch) Put(op *core.OutPoint, addr common.Uint168) error { +func (b *opsBatch) Put(op *util.OutPoint, addr common.Uint168) error { b.Lock() defer b.Unlock() return b.Tx.Bucket(BKTOps).Put(op.Bytes(), addr.Bytes()) } -func (b *opsBatch) Del(op *core.OutPoint) error { +func (b *opsBatch) Del(op *util.OutPoint) error { b.Lock() defer b.Unlock() return b.Tx.Bucket(BKTOps).Delete(op.Bytes()) diff --git a/interface/store/txs.go b/interface/store/txs.go index 249a68a..cf55e6e 100644 --- a/interface/store/txs.go +++ b/interface/store/txs.go @@ -55,7 +55,7 @@ func (t *txs) Put(txn *util.Tx) (err error) { return err } - if err = tx.Bucket(BKTTxs).Put(txn.Hash().Bytes(), buf.Bytes()); err != nil { + if err = tx.Bucket(BKTTxs).Put(txn.Hash.Bytes(), buf.Bytes()); err != nil { return err } @@ -66,7 +66,7 @@ func (t *txs) Put(txn *util.Tx) (err error) { var txMap = make(map[common.Uint256]uint32) gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) - txMap[txn.Hash()] = txn.Height + txMap[txn.Hash] = txn.Height buf = new(bytes.Buffer) if err = gob.NewEncoder(buf).Encode(txMap); err != nil { diff --git a/interface/store/txsbatch.go b/interface/store/txsbatch.go index 6427c21..11e5ee1 100644 --- a/interface/store/txsbatch.go +++ b/interface/store/txsbatch.go @@ -31,7 +31,7 @@ func (b *txsBatch) Put(tx *util.Tx) error { return err } - err := b.Tx.Bucket(BKTTxs).Put(tx.Hash().Bytes(), buf.Bytes()) + err := b.Tx.Bucket(BKTTxs).Put(tx.Hash.Bytes(), buf.Bytes()) if err != nil { return err } @@ -103,7 +103,7 @@ func (b *txsBatch) Commit() error { gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) for _, tx := range txs { - txMap[tx.Hash()] = height + txMap[tx.Hash] = height } var buf = new(bytes.Buffer) @@ -130,7 +130,7 @@ func (b *txsBatch) Commit() error { } for _, tx := range txs { - delete(txMap, tx.Hash()) + delete(txMap, tx.Hash) } var buf = new(bytes.Buffer) diff --git a/log.go b/log.go index 405685d..f2177b9 100644 --- a/log.go +++ b/log.go @@ -7,13 +7,12 @@ import ( "github.com/elastos/Elastos.ELA.SPV/blockchain" "github.com/elastos/Elastos.ELA.SPV/peer" "github.com/elastos/Elastos.ELA.SPV/sdk" - "github.com/elastos/Elastos.ELA.SPV/spvwallet" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/store" "github.com/elastos/Elastos.ELA.SPV/sync" + "github.com/elastos/Elastos.ELA.SPV/wallet" + "github.com/elastos/Elastos.ELA.SPV/wallet/store" "github.com/elastos/Elastos.ELA.Utility/elalog" + "github.com/elastos/Elastos.ELA.Utility/http/jsonrpc" "github.com/elastos/Elastos.ELA.Utility/p2p/addrmgr" "github.com/elastos/Elastos.ELA.Utility/p2p/connmgr" "github.com/elastos/Elastos.ELA.Utility/p2p/server" @@ -27,10 +26,10 @@ const LogPath = "./logs-spv/" var ( fileWriter = elalog.NewFileWriter( LogPath, - config.Values().MaxPerLogSize, - config.Values().MaxLogsSize, + config.MaxPerLogSize, + config.MaxLogsSize, ) - level = elalog.Level(config.Values().PrintLevel) + level = elalog.Level(config.PrintLevel) backend = elalog.NewBackend(io.MultiWriter(os.Stdout, fileWriter), elalog.Llongfile) @@ -50,10 +49,10 @@ func init() { connmgr.UseLogger(cmgrlog) blockchain.UseLogger(bcdblog) sdk.UseLogger(spvslog) - rpc.UseLogger(rpcslog) + jsonrpc.UseLogger(rpcslog) peer.UseLogger(peerlog) server.UseLogger(srvrlog) store.UseLogger(bcdblog) sync.UseLogger(synclog) - spvwallet.UseLogger(waltlog) + wallet.UseLogger(waltlog) } diff --git a/main.go b/main.go index aa0f6dd..fa70298 100644 --- a/main.go +++ b/main.go @@ -3,13 +3,11 @@ package main import ( "os" "os/signal" - - "github.com/elastos/Elastos.ELA.SPV/spvwallet" ) func main() { // Initiate SPV service - wallet, err := spvwallet.New() + w, err := NewWallet() if err != nil { waltlog.Error("Initiate SPV service failed,", err) os.Exit(0) @@ -22,12 +20,12 @@ func main() { go func() { for range c { waltlog.Trace("Wallet shutting down...") - wallet.Stop() + w.Stop() stop <- 1 } }() - wallet.Start() + w.Start() <-stop } diff --git a/peer/peer.go b/peer/peer.go index 4aa88c4..0b1158e 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -5,14 +5,13 @@ import ( "sync" "time" + "github.com/elastos/Elastos.ELA.SPV/bloom" "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA.Utility/p2p/peer" - "github.com/elastos/Elastos.ELA/bloom" - "github.com/elastos/Elastos.ELA/core" ) const ( @@ -43,7 +42,7 @@ type Config struct { // After sent a data request with invType TRANSACTION, a txn message will return through this method. // these transactions are matched to the bloom filter you have sent with the filterload message. - OnTx func(*Peer, *core.Transaction) + OnTx func(*Peer, util.Transaction) // If the BLOCK or TRANSACTION requested by the data request message can not be found, // notfound message with requested data hash will return through this method. @@ -74,33 +73,28 @@ type Peer struct { prevGetBlocksBegin *common.Uint256 prevGetBlocksStop *common.Uint256 - stallControl chan interface{} - blockQueue chan interface{} - outputQueue chan outMsg - queueQuit chan struct{} + stallControl chan interface{} + blockQueue chan interface{} + outputQueue chan outMsg + sendDoneQueue chan struct{} } func NewPeer(peer *peer.Peer, cfg *Config) *Peer { p := Peer{ - Peer: peer, - cfg: *cfg, - stallControl: make(chan interface{}, 1), - blockQueue: make(chan interface{}, 1), - outputQueue: make(chan outMsg, outputBufferSize), - queueQuit: make(chan struct{}), + Peer: peer, + cfg: *cfg, + stallControl: make(chan interface{}, 1), + blockQueue: make(chan interface{}, 1), + sendDoneQueue: make(chan struct{}, 1), + outputQueue: make(chan outMsg, outputBufferSize), } peer.AddMessageFunc(p.handleMessage) + peer.OnSendDone(p.sendDoneQueue) go p.stallHandler() go p.queueHandler() go p.blockHandler() - go func() { - // We have waited on queueQuit and thus we can be sure - // that we will not miss anything sent on sendQueue. - <-p.queueQuit - p.CleanupSendQueue() - }() return &p } @@ -180,11 +174,11 @@ out: case *msg.MerkleBlock: // Remove received merkleblock from expected response map. - delete(pendingResponses, m.Header.(*core.Header).Hash().String()) + delete(pendingResponses, m.Header.(util.BlockHeader).Hash().String()) case *msg.Tx: // Remove received transaction from expected response map. - delete(pendingResponses, m.Serializable.(*core.Transaction).Hash().String()) + delete(pendingResponses, m.Serializable.(util.Transaction).Hash().String()) case *msg.NotFound: // NotFound should not received from sync peer @@ -248,7 +242,7 @@ func (p *Peer) blockHandler() { // Data caches for the downloading block. var header *util.Header var pendingTxs map[common.Uint256]struct{} - var txs []*core.Transaction + var txs []util.Transaction // NotifyOnBlock message and clear cached data. notifyBlock := func() { @@ -294,10 +288,10 @@ out: // Set current downloading block header = &util.Header{ - Header: *m.Header.(*core.Header), - NumTxs: m.Transactions, - Hashes: m.Hashes, - Flags: m.Flags, + BlockHeader: m.Header.(util.BlockHeader), + NumTxs: m.Transactions, + Hashes: m.Hashes, + Flags: m.Flags, } // No transaction included in this block, so just notify block @@ -314,11 +308,11 @@ out: } // Initiate transactions cache. - txs = make([]*core.Transaction, 0, len(pendingTxs)) + txs = make([]util.Transaction, 0, len(pendingTxs)) case *msg.Tx: // Not in block downloading mode, just notify new transaction. - tx := m.Serializable.(*core.Transaction) + tx := m.Serializable.(util.Transaction) if header == nil { p.cfg.OnTx(p, tx) continue @@ -398,7 +392,7 @@ out: // This channel is notified when a message has been sent across // the network socket. - case <-p.SendDoneQueue(): + case <-p.sendDoneQueue: // No longer waiting if there are no more messages // in the pending messages queue. next := pendingMsgs.Front() @@ -437,7 +431,6 @@ cleanup: break cleanup } } - close(p.queueQuit) log.Tracef("Peer queue handler done for %s", p) } diff --git a/sdk/bloom.go b/sdk/bloom.go deleted file mode 100644 index 75fcc36..0000000 --- a/sdk/bloom.go +++ /dev/null @@ -1,46 +0,0 @@ -package sdk - -import ( - "github.com/elastos/Elastos.ELA.SPV/fprate" - - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/bloom" - "github.com/elastos/Elastos.ELA/core" -) - -// Create a new bloom filter instance -// elements are how many elements will be added to this filter. -func NewBloomFilter(elements uint32) *bloom.Filter { - return bloom.NewFilter(elements, 0, fprate.ReducedFalsePositiveRate) -} - -// Build a bloom filter by giving the interested addresses and outpoints -func BuildBloomFilter(addresses []*common.Uint168, outpoints []*core.OutPoint) *bloom.Filter { - elements := uint32(len(addresses) + len(outpoints)) - - filter := NewBloomFilter(elements) - for _, address := range addresses { - filter.Add(address.Bytes()) - } - - for _, outpoint := range outpoints { - filter.AddOutPoint(outpoint) - } - - return filter -} - -// Add a address into the given bloom filter -func FilterAddress(filter *bloom.Filter, address *common.Uint168) { - filter.Add(address.Bytes()) -} - -// Add a account into the given bloom filter -func FilterAccount(filter *bloom.Filter, account *Account) { - filter.Add(account.programHash.Bytes()) -} - -// Add a outpoint into the given bloom filter -func FilterOutpoint(filter *bloom.Filter, op *core.OutPoint) { - filter.AddOutPoint(op) -} diff --git a/sdk/interface.go b/sdk/interface.go index c9f8578..ed63569 100644 --- a/sdk/interface.go +++ b/sdk/interface.go @@ -5,7 +5,6 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" ) /* @@ -29,22 +28,22 @@ type IService interface { UpdateFilter() // SendTransaction broadcast a transaction message to the peer to peer network. - SendTransaction(core.Transaction) error + SendTransaction(util.Transaction) error } // StateNotifier exposes methods to notify status changes of transactions and blocks. type StateNotifier interface { // TransactionAnnounce will be invoked when received a new announced transaction. - TransactionAnnounce(tx *core.Transaction) + TransactionAnnounce(tx util.Transaction) // TransactionAccepted will be invoked after a transaction sent by // SendTransaction() method has been accepted. Notice: this method needs at // lest two connected peers to work. - TransactionAccepted(tx *core.Transaction) + TransactionAccepted(tx util.Transaction) // TransactionRejected will be invoked if a transaction sent by SendTransaction() // method has been rejected. - TransactionRejected(tx *core.Transaction) + TransactionRejected(tx util.Transaction) // TransactionConfirmed will be invoked after a transaction sent by // SendTransaction() method has been packed into a block. @@ -72,12 +71,18 @@ type Config struct { // The min candidate peers count to start syncing progress. MinPeersForSync int - // Foundation address of the current access blockhain network - Foundation string + // GenesisHeader is the + GenesisHeader util.BlockHeader // The database to store all block headers ChainStore database.ChainStore + // NewTransaction create a new transaction instance. + NewTransaction func() util.Transaction + + // NewBlockHeader create a new block header instance. + NewBlockHeader func() util.BlockHeader + // GetFilterData() returns two arguments. // First arguments are all addresses stored in your data store. // Second arguments are all balance references to those addresses stored in your data store, @@ -86,7 +91,7 @@ type Config struct { // reference of an transaction output. If an address ever received an transaction output, // there will be the outpoint reference to it. Any time you want to spend the balance of an // address, you must provide the reference of the balance which is an outpoint in the transaction input. - GetFilterData func() ([]*common.Uint168, []*core.OutPoint) + GetFilterData func() ([]*common.Uint168, []*util.OutPoint) // StateNotifier is an optional config, if you don't want to receive state changes of transactions // or blocks, just keep it blank. diff --git a/sdk/service.go b/sdk/service.go index b02b7bb..cd1392e 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -5,7 +5,8 @@ import ( "time" "github.com/elastos/Elastos.ELA.SPV/blockchain" - spvpeer "github.com/elastos/Elastos.ELA.SPV/peer" + "github.com/elastos/Elastos.ELA.SPV/bloom" + speer "github.com/elastos/Elastos.ELA.SPV/peer" "github.com/elastos/Elastos.ELA.SPV/sync" "github.com/elastos/Elastos.ELA.SPV/util" @@ -14,8 +15,6 @@ import ( "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA.Utility/p2p/peer" "github.com/elastos/Elastos.ELA.Utility/p2p/server" - "github.com/elastos/Elastos.ELA/bloom" - "github.com/elastos/Elastos.ELA/core" ) const ( @@ -24,7 +23,7 @@ const ( ) type sendTxMsg struct { - tx *core.Transaction + tx util.Transaction expire time.Time } @@ -58,18 +57,18 @@ type service struct { // Create a instance of SPV service implementation. func newService(cfg *Config) (*service, error) { // Initialize blockchain - chain, err := blockchain.New(cfg.Foundation, cfg.ChainStore) + chain, err := blockchain.New(cfg.GenesisHeader, cfg.ChainStore) if err != nil { return nil, err } // Create SPV service instance service := &service{ - cfg: *cfg, - newPeers: make(chan *peer.Peer, cfg.MaxPeers), - donePeers: make(chan *peer.Peer, cfg.MaxPeers), - txQueue: make(chan interface{}, 3), - quit: make(chan struct{}), + cfg: *cfg, + newPeers: make(chan *peer.Peer, cfg.MaxPeers), + donePeers: make(chan *peer.Peer, cfg.MaxPeers), + txQueue: make(chan interface{}, 3), + quit: make(chan struct{}), txProcessed: make(chan struct{}, 1), blockProcessed: make(chan struct{}, 1), } @@ -132,7 +131,7 @@ func (s *service) start() { } func (s *service) updateFilter() *bloom.Filter { - return BuildBloomFilter(s.cfg.GetFilterData()) + return bloom.BuildBloomFilter(s.cfg.GetFilterData()) } func (s *service) makeEmptyMessage(cmd string) (p2p.Message, error) { @@ -148,10 +147,10 @@ func (s *service) makeEmptyMessage(cmd string) (p2p.Message, error) { message = new(msg.NotFound) case p2p.CmdTx: - message = msg.NewTx(new(core.Transaction)) + message = msg.NewTx(s.cfg.NewTransaction()) case p2p.CmdMerkleBlock: - message = msg.NewMerkleBlock(new(core.Header)) + message = msg.NewMerkleBlock(s.cfg.NewBlockHeader()) case p2p.CmdReject: message = new(msg.Reject) @@ -175,15 +174,15 @@ func (s *service) donePeer(peer server.IPeer) { // peerHandler handles new peers and done peers from P2P server. // When comes new peer, create a spv peer warpper for it func (s *service) peerHandler() { - peers := make(map[*peer.Peer]*spvpeer.Peer) + peers := make(map[*peer.Peer]*speer.Peer) out: for { select { case p := <-s.newPeers: // Create spv peer warpper for the new peer. - sp := spvpeer.NewPeer(p, - &spvpeer.Config{ + sp := speer.NewPeer(p, + &speer.Config{ OnInv: s.onInv, OnTx: s.onTx, OnBlock: s.onBlock, @@ -264,7 +263,7 @@ out: accepted[txId] = struct{}{} // Use a new goroutine do the invoke to prevent blocking. - go func(tx *core.Transaction) { + go func(tx util.Transaction) { if s.cfg.StateNotifier != nil { s.cfg.StateNotifier.TransactionAccepted(tx) } @@ -289,7 +288,7 @@ out: delete(unconfirmed, txId) // Use a new goroutine do the invoke to prevent blocking. - go func(tx *core.Transaction) { + go func(tx util.Transaction) { if s.cfg.StateNotifier != nil { s.cfg.StateNotifier.TransactionRejected(tx) } @@ -299,7 +298,7 @@ out: case *blockMsg: // Loop through all packed transactions, see if match to any // sent transactions. - confirmedTxs := make(map[common.Uint256]*core.Transaction) + confirmedTxs := make(map[common.Uint256]util.Transaction) for _, tx := range tmsg.block.Transactions { txId := tx.Hash() @@ -328,7 +327,7 @@ out: if s.cfg.StateNotifier != nil { s.cfg.StateNotifier.TransactionConfirmed(tx) } - }(util.NewTx(*tx, tmsg.block.Height)) + }(util.NewTx(tx, tmsg.block.Height)) } } case <-retryTicker.C: @@ -363,7 +362,7 @@ cleanup: log.Trace("Service transaction handler done") } -func (s *service) SendTransaction(tx core.Transaction) error { +func (s *service) SendTransaction(tx util.Transaction) error { peersCount := s.IServer.ConnectedCount() if peersCount < int32(s.cfg.MinPeersForSync) { return fmt.Errorf("connected peers %d not enough for sending transactions", peersCount) @@ -373,11 +372,11 @@ func (s *service) SendTransaction(tx core.Transaction) error { return fmt.Errorf("spv service did not sync to current") } - s.txQueue <- &sendTxMsg{tx: &tx} + s.txQueue <- &sendTxMsg{tx: tx} return nil } -func (s *service) onInv(sp *spvpeer.Peer, inv *msg.Inv) { +func (s *service) onInv(sp *speer.Peer, inv *msg.Inv) { // If service already synced to current, it most likely to receive a relayed // block or transaction inv, not a huge invList with block hashes. if s.IsCurrent() { @@ -391,7 +390,7 @@ func (s *service) onInv(sp *spvpeer.Peer, inv *msg.Inv) { s.syncManager.QueueInv(inv, sp) } -func (s *service) onBlock(sp *spvpeer.Peer, block *util.Block) { +func (s *service) onBlock(sp *speer.Peer, block *util.Block) { s.syncManager.QueueBlock(block, sp, s.blockProcessed) select { @@ -403,12 +402,12 @@ func (s *service) onBlock(sp *spvpeer.Peer, block *util.Block) { } } -func (s *service) onTx(sp *spvpeer.Peer, msgTx *core.Transaction) { +func (s *service) onTx(sp *speer.Peer, msgTx util.Transaction) { s.syncManager.QueueTx(msgTx, sp, s.txProcessed) <-s.txProcessed } -func (s *service) onNotFound(sp *spvpeer.Peer, notFound *msg.NotFound) { +func (s *service) onNotFound(sp *speer.Peer, notFound *msg.NotFound) { // Every thing we requested was came from this connected peer, so // no reason it said I have some data you don't have and when you // come to get it, it say oh I didn't have it. @@ -416,7 +415,7 @@ func (s *service) onNotFound(sp *spvpeer.Peer, notFound *msg.NotFound) { sp.Disconnect() } -func (s *service) onReject(sp *spvpeer.Peer, reject *msg.Reject) { +func (s *service) onReject(sp *speer.Peer, reject *msg.Reject) { if reject.Cmd == p2p.CmdTx { s.txQueue <- &txRejectMsg{iv: &msg.InvVect{Type: msg.InvTypeTx, Hash: reject.Hash}} } diff --git a/spvwallet.go b/spvwallet.go new file mode 100644 index 0000000..bfcf5ae --- /dev/null +++ b/spvwallet.go @@ -0,0 +1,399 @@ +package main + +import ( + "bytes" + "encoding/hex" + "fmt" + "time" + + "github.com/elastos/Elastos.ELA.SPV/database" + "github.com/elastos/Elastos.ELA.SPV/sdk" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.SPV/wallet/store/headers" + "github.com/elastos/Elastos.ELA.SPV/wallet/store/sqlite" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/crypto" + httputil "github.com/elastos/Elastos.ELA.Utility/http/util" + "github.com/elastos/Elastos.ELA/core" +) + +const ( + MaxPeers = 12 + MinPeersForSync = 2 +) + +var ErrInvalidParameter = fmt.Errorf("invalide parameter") + +type spvwallet struct { + sdk.IService + db sqlite.DataStore + filter *sdk.AddrFilter +} + +// Batch returns a TxBatch instance for transactions batch +// commit, this can get better performance when commit a bunch +// of transactions within a block. +func (w *spvwallet) Batch() database.TxBatch { + return &txBatch{ + db: w.db, + batch: w.db.Batch(), + filter: w.getAddrFilter(), + } +} + +// HaveTx returns if the transaction already saved in database +// by it's id. +func (w *spvwallet) HaveTx(txId *common.Uint256) (bool, error) { + tx, err := w.db.Txs().Get(txId) + return tx != nil, err +} + +// GetTxs returns all transactions within the given height. +func (w *spvwallet) GetTxs(height uint32) ([]*util.Tx, error) { + return nil, nil +} + +// RemoveTxs delete all transactions on the given height. Return +// how many transactions are deleted from database. +func (w *spvwallet) RemoveTxs(height uint32) (int, error) { + batch := w.db.Batch() + err := batch.RollbackHeight(height) + if err != nil { + return 0, batch.Rollback() + } + return 0, batch.Commit() +} + +// Clear delete all data in database. +func (w *spvwallet) Clear() error { + return w.db.Clear() +} + +// Close database. +func (w *spvwallet) Close() error { + return w.db.Close() +} + +func (w *spvwallet) GetFilterData() ([]*common.Uint168, []*util.OutPoint) { + utxos, err := w.db.UTXOs().GetAll() + if err != nil { + waltlog.Debugf("GetAll UTXOs error: %v", err) + } + stxos, err := w.db.STXOs().GetAll() + if err != nil { + waltlog.Debugf("GetAll STXOs error: %v", err) + } + outpoints := make([]*util.OutPoint, 0, len(utxos)+len(stxos)) + for _, utxo := range utxos { + outpoints = append(outpoints, utxo.Op) + } + for _, stxo := range stxos { + outpoints = append(outpoints, stxo.Op) + } + + return w.getAddrFilter().GetAddrs(), outpoints +} + +func (w *spvwallet) NotifyNewAddress(hash []byte) { + // Reload address filter to include new address + w.loadAddrFilter() + // Broadcast filterload message to connected peers + w.UpdateFilter() +} + +func (w *spvwallet) getAddrFilter() *sdk.AddrFilter { + if w.filter == nil { + w.loadAddrFilter() + } + return w.filter +} + +func (w *spvwallet) loadAddrFilter() *sdk.AddrFilter { + addrs, _ := w.db.Addrs().GetAll() + w.filter = sdk.NewAddrFilter(nil) + for _, addr := range addrs { + w.filter.AddAddr(addr.Hash()) + } + return w.filter +} + +// TransactionAnnounce will be invoked when received a new announced transaction. +func (w *spvwallet) TransactionAnnounce(tx util.Transaction) { + // TODO +} + +// TransactionAccepted will be invoked after a transaction sent by +// SendTransaction() method has been accepted. Notice: this method needs at +// lest two connected peers to work. +func (w *spvwallet) TransactionAccepted(tx util.Transaction) { + // TODO +} + +// TransactionRejected will be invoked if a transaction sent by SendTransaction() +// method has been rejected. +func (w *spvwallet) TransactionRejected(tx util.Transaction) { + // TODO +} + +// TransactionConfirmed will be invoked after a transaction sent by +// SendTransaction() method has been packed into a block. +func (w *spvwallet) TransactionConfirmed(tx *util.Tx) { + // TODO +} + +// BlockCommitted will be invoked when a block and transactions within it are +// successfully committed into database. +func (w *spvwallet) BlockCommitted(block *util.Block) { + if !w.IsCurrent() { + return + } + + w.db.State().PutHeight(block.Height) + // TODO +} + +type txBatch struct { + db sqlite.DataStore + batch sqlite.DataBatch + filter *sdk.AddrFilter +} + +// PutTx add a store transaction operation into batch, and return +// if it is a false positive and error. +func (b *txBatch) PutTx(mtx util.Transaction, height uint32) (bool, error) { + tx := mtx.(*core.Transaction) + txId := tx.Hash() + hits := 0 + + // Check if any UTXOs within this wallet have been spent. + for _, input := range tx.Inputs { + // Move UTXO to STXO + op := util.NewOutPoint(input.Previous.TxID, input.Previous.Index) + utxo, _ := b.db.UTXOs().Get(op) + // Skip if no match. + if utxo == nil { + continue + } + + err := b.batch.STXOs().Put(sutil.NewSTXO(utxo, height, txId)) + if err != nil { + return false, nil + } + hits++ + } + + // Check if there are any output to this wallet address. + for index, output := range tx.Outputs { + // Filter address + if b.filter.ContainAddr(output.ProgramHash) { + var lockTime = output.OutputLock + if tx.TxType == core.CoinBase { + lockTime = height + 100 + } + utxo := sutil.NewUTXO(txId, height, index, output.Value, lockTime, output.ProgramHash) + err := b.batch.UTXOs().Put(utxo) + if err != nil { + return false, err + } + hits++ + } + } + + // If no hits, no need to save transaction + if hits == 0 { + return true, nil + } + + // Save transaction + err := b.batch.Txs().Put(util.NewTx(tx, height)) + if err != nil { + return false, err + } + + return false, nil +} + +// DelTx add a delete transaction operation into batch. +func (b *txBatch) DelTx(txId *common.Uint256) error { + return b.batch.Txs().Del(txId) +} + +// DelTxs add a delete transactions on given height operation. +func (b *txBatch) DelTxs(height uint32) error { + // Delete transactions is used when blockchain doing rollback, this not + // only delete the transactions on the given height, and also restore + // STXOs and remove UTXOs within these transactions. + return b.batch.RollbackHeight(height) +} + +// Rollback cancel all operations in current batch. +func (b *txBatch) Rollback() error { + return b.batch.Rollback() +} + +// Commit the added transactions into database. +func (b *txBatch) Commit() error { + return b.batch.Commit() +} + +// Functions for RPC service. +func (w *spvwallet) notifyNewAddress(params httputil.Params) (interface{}, error) { + data, ok := params.String("addr") + if !ok { + return nil, ErrInvalidParameter + } + + _, err := hex.DecodeString(data) + if err != nil { + return nil, err + } + + // Reload address filter to include new address + w.loadAddrFilter() + + // Broadcast filterload message to connected peers + w.UpdateFilter() + + return nil, nil +} + +func (w *spvwallet) sendTransaction(params httputil.Params) (interface{}, error) { + data, ok := params.String("data") + if !ok { + return nil, ErrInvalidParameter + } + + txBytes, err := hex.DecodeString(data) + if err != nil { + return nil, ErrInvalidParameter + } + + var tx core.Transaction + err = tx.Deserialize(bytes.NewReader(txBytes)) + if err != nil { + return nil, fmt.Errorf("deserialize transaction failed %s", err) + } + + return nil, w.SendTransaction(&tx) +} + +func NewWallet() (*spvwallet, error) { + // Initialize headers db + headers, err := headers.NewDatabase(newBlockHeader) + if err != nil { + return nil, err + } + + db, err := sqlite.NewDatabase() + if err != nil { + return nil, err + } + + w := spvwallet{ + db: db, + } + chainStore := database.NewDefaultChainDB(headers, &w) + + // Initialize spv service + w.IService, err = sdk.NewService( + &sdk.Config{ + Magic: config.Magic, + SeedList: config.SeedList, + DefaultPort: config.NodePort, + MaxPeers: MaxPeers, + MinPeersForSync: MinPeersForSync, + GenesisHeader: GenesisHeader(), + ChainStore: chainStore, + NewTransaction: newTransaction, + NewBlockHeader: newBlockHeader, + GetFilterData: w.GetFilterData, + StateNotifier: &w, + }) + if err != nil { + return nil, err + } + + return &w, nil +} + +func newTransaction() util.Transaction { + return new(core.Transaction) +} + +// GenesisHeader creates a specific genesis header by the given +// foundation address. +func GenesisHeader() util.BlockHeader { + // Genesis time + genesisTime := time.Date(2017, time.December, 22, 10, 0, 0, 0, time.UTC) + + // header + header := core.Header{ + Version: core.BlockVersion, + Previous: common.EmptyHash, + MerkleRoot: common.EmptyHash, + Timestamp: uint32(genesisTime.Unix()), + Bits: 0x1d03ffff, + Nonce: core.GenesisNonce, + Height: uint32(0), + } + + // ELA coin + elaCoin := &core.Transaction{ + TxType: core.RegisterAsset, + PayloadVersion: 0, + Payload: &core.PayloadRegisterAsset{ + Asset: core.Asset{ + Name: "ELA", + Precision: 0x08, + AssetType: 0x00, + }, + Amount: 0 * 100000000, + Controller: common.Uint168{}, + }, + Attributes: []*core.Attribute{}, + Inputs: []*core.Input{}, + Outputs: []*core.Output{}, + Programs: []*core.Program{}, + } + + coinBase := &core.Transaction{ + TxType: core.CoinBase, + PayloadVersion: core.PayloadCoinBaseVersion, + Payload: new(core.PayloadCoinBase), + Inputs: []*core.Input{ + { + Previous: core.OutPoint{ + TxID: common.EmptyHash, + Index: 0x0000, + }, + Sequence: 0x00000000, + }, + }, + Attributes: []*core.Attribute{}, + LockTime: 0, + Programs: []*core.Program{}, + } + + coinBase.Outputs = []*core.Output{ + { + AssetID: elaCoin.Hash(), + Value: 3300 * 10000 * 100000000, + ProgramHash: *config.foundation, + }, + } + + nonce := []byte{0x4d, 0x65, 0x82, 0x21, 0x07, 0xfc, 0xfd, 0x52} + txAttr := core.NewAttribute(core.Nonce, nonce) + coinBase.Attributes = append(coinBase.Attributes, &txAttr) + + transactions := []*core.Transaction{coinBase, elaCoin} + hashes := make([]common.Uint256, 0, len(transactions)) + for _, tx := range transactions { + hashes = append(hashes, tx.Hash()) + } + header.MerkleRoot, _ = crypto.ComputeRoot(hashes) + + return &blockHeader{Header: &header} +} diff --git a/spvwallet/config/config.go b/spvwallet/config/config.go deleted file mode 100644 index 2705da0..0000000 --- a/spvwallet/config/config.go +++ /dev/null @@ -1,51 +0,0 @@ -package config - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" -) - -const ( - ConfigFilename = "./config.json" -) - -var config *Config // The single instance of config - -type Config struct { - Magic uint32 - SeedList []string - DefaultPort uint16 // default port for public peers to provide services. - PrintLevel int - MaxLogsSize int64 - MaxPerLogSize int64 - RPCPort uint16 - Foundation string -} - -func (config *Config) readConfigFile() error { - data, err := ioutil.ReadFile(ConfigFilename) - if err != nil { - return err - } - // Remove the UTF-8 Byte Order Mark - data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf")) - - err = json.Unmarshal(data, config) - if err != nil { - return err - } - return nil -} - -func Values() *Config { - if config == nil { - config = new(Config) - err := config.readConfigFile() - if err != nil { - fmt.Println("Read config file error:", err) - } - } - return config -} diff --git a/spvwallet/log.go b/spvwallet/log.go deleted file mode 100644 index 13998e8..0000000 --- a/spvwallet/log.go +++ /dev/null @@ -1,28 +0,0 @@ -package spvwallet - -import ( - "github.com/elastos/Elastos.ELA.Utility/elalog" -) - -// log is a logger that is initialized with no output filters. This -// means the package will not perform any logging by default until the caller -// requests it. -var log elalog.Logger - -// The default amount of logging is none. -func init() { - DisableLog() -} - -// DisableLog disables all library log output. Logging output is disabled -// by default until either UseLogger or SetLogWriter are called. -func DisableLog() { - log = elalog.Disabled -} - -// UseLogger uses a specified Logger to output package logging info. -// This should be used in preference to SetLogWriter if the caller is also -// using elalog. -func UseLogger(logger elalog.Logger) { - log = logger -} diff --git a/spvwallet/rpc/client.go b/spvwallet/rpc/client.go deleted file mode 100644 index 7e64483..0000000 --- a/spvwallet/rpc/client.go +++ /dev/null @@ -1,73 +0,0 @@ -package rpc - -import ( - "bytes" - "encoding/hex" - "encoding/json" - "errors" - "io/ioutil" - "net/http" - - "github.com/elastos/Elastos.ELA/core" -) - -type Client struct { - url string -} - -func GetClient() *Client { - return &Client{url: RPCAddr} -} - -func (client *Client) NotifyNewAddress(hash []byte) error { - resp := client.send( - &Req{ - Method: "notifynewaddress", - Params: []interface{}{hex.EncodeToString(hash)}, - }, - ) - if resp.Code != 0 { - return errors.New(resp.Result.(string)) - } - return nil -} - -func (client *Client) SendTransaction(tx *core.Transaction) error { - buf := new(bytes.Buffer) - tx.Serialize(buf) - resp := client.send( - &Req{ - Method: "sendtransaction", - Params: []interface{}{hex.EncodeToString(buf.Bytes())}, - }, - ) - if resp.Code != 0 { - return errors.New(resp.Result.(string)) - } - return nil -} - -func (client *Client) send(req *Req) (ret Resp) { - data, err := json.Marshal(req) - if err != nil { - return MarshalRequestError - } - - resp, err := http.Post(client.url, "application/json", bytes.NewReader(data)) - if err != nil { - return PostRequestError - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return ReadResponseError - } - - err = json.Unmarshal(body, &ret) - if err != nil { - return UnmarshalResponseError - } - - return ret -} diff --git a/spvwallet/rpc/fucntions.go b/spvwallet/rpc/fucntions.go deleted file mode 100644 index 99068f1..0000000 --- a/spvwallet/rpc/fucntions.go +++ /dev/null @@ -1,42 +0,0 @@ -package rpc - -import ( - "bytes" - "encoding/hex" - - "github.com/elastos/Elastos.ELA/core" -) - -func (server *Server) notifyNewAddress(req Req) Resp { - data, ok := req.Params[0].(string) - if !ok { - return InvalidParameter - } - addr, err := hex.DecodeString(data) - if err != nil { - return FunctionError(err.Error()) - } - server.NotifyNewAddress(addr) - return Success("New address received") -} - -func (server *Server) sendTransaction(req Req) Resp { - data, ok := req.Params[0].(string) - if !ok { - return InvalidParameter - } - txBytes, err := hex.DecodeString(data) - if err != nil { - return FunctionError(err.Error()) - } - var tx core.Transaction - err = tx.Deserialize(bytes.NewReader(txBytes)) - if err != nil { - return FunctionError("Deserialize transaction failed") - } - err = server.SendTransaction(tx) - if err != nil { - return FunctionError(err.Error()) - } - return Success(tx.Hash().String()) -} diff --git a/spvwallet/rpc/protocol.go b/spvwallet/rpc/protocol.go deleted file mode 100644 index f42e302..0000000 --- a/spvwallet/rpc/protocol.go +++ /dev/null @@ -1,46 +0,0 @@ -package rpc - -import ( - "fmt" - - "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" -) - -var ( - RPCPort = config.Values().RPCPort - RPCAddr = fmt.Sprint("http://127.0.0.1:", RPCPort, "/spvwallet/") -) - -type Req struct { - Method string `json:"method"` - Params []interface{} `json:"params"` -} - -type Resp struct { - Code int `json:"code"` - Result interface{} `json:"result"` -} - -var ( - MarshalRequestError = Resp{301, "MarshalRequestError"} - PostRequestError = Resp{302, "PostRequestError"} - ReadResponseError = Resp{303, "ReadResponseError"} - UnmarshalResponseError = Resp{304, "UnmarshalResponseError"} -) - -var ( - NonPostRequest = Resp{401, "NonPostRequest"} - EmptyRequestBody = Resp{402, "EmptyRequestBody"} - ReadRequestError = Resp{403, "ReadRequestError"} - UnmarshalRequestError = Resp{404, "UnmarshalRequestError"} - InvalidMethod = Resp{405, "InvalidMethod"} - InvalidParameter = Resp{406, "InvalidParameter"} -) - -func Success(result interface{}) Resp { - return Resp{0, result} -} - -func FunctionError(error string) Resp { - return Resp{407, error} -} diff --git a/spvwallet/rpc/server.go b/spvwallet/rpc/server.go deleted file mode 100644 index dcdb905..0000000 --- a/spvwallet/rpc/server.go +++ /dev/null @@ -1,78 +0,0 @@ -package rpc - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "os" - - "github.com/elastos/Elastos.ELA/core" -) - -func InitServer() *Server { - server := new(Server) - server.Server = http.Server{Addr: fmt.Sprint(":", RPCPort)} - server.methods = map[string]func(Req) Resp{ - "notifynewaddress": server.notifyNewAddress, - "sendtransaction": server.sendTransaction, - } - http.HandleFunc("/spvwallet/", server.handle) - return server -} - -type Server struct { - http.Server - methods map[string]func(Req) Resp - NotifyNewAddress func([]byte) - SendTransaction func(core.Transaction) error -} - -func (server *Server) Start() { - go func() { - err := server.ListenAndServe() - if err != nil { - log.Error("RPC service start failed:", err) - os.Exit(800) - } - }() - log.Debug("RPC server started...") -} - -func (server *Server) handle(w http.ResponseWriter, r *http.Request) { - resp := server.getResp(r) - data, err := json.Marshal(resp) - if err != nil { - log.Error("Marshal response error: ", err) - } - w.Write(data) -} - -func (server *Server) getResp(r *http.Request) Resp { - if r.Method != "POST" { - return NonPostRequest - } - - if r.Body == nil { - return EmptyRequestBody - } - - //read the body of the request - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return ReadRequestError - } - - var req Req - err = json.Unmarshal(body, &req) - if err != nil { - return UnmarshalRequestError - } - - function, ok := server.methods[req.Method] - if !ok { - return InvalidMethod - } - - return function(req) -} diff --git a/spvwallet/sutil/tx.go b/spvwallet/sutil/tx.go deleted file mode 100644 index 1d06b0e..0000000 --- a/spvwallet/sutil/tx.go +++ /dev/null @@ -1,40 +0,0 @@ -package sutil - -import ( - "fmt" - "time" - - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" -) - -type Tx struct { - // Transaction ID - TxId common.Uint256 - - // The height at which it was mined - Height uint32 - - // The time the transaction was first seen - Timestamp time.Time - - // Transaction - Data core.Transaction -} - -func NewTx(tx core.Transaction, height uint32) *Tx { - storeTx := new(Tx) - storeTx.TxId = tx.Hash() - storeTx.Height = height - storeTx.Timestamp = time.Now() - storeTx.Data = tx - return storeTx -} - -func (tx *Tx) String() string { - return fmt.Sprintln( - "{TxId:", tx.TxId.String(), - ", Height:", tx.Height, - ", Timestamp:", tx.Timestamp, - "}") -} diff --git a/spvwallet/wallet.go b/spvwallet/wallet.go deleted file mode 100644 index 018d1c7..0000000 --- a/spvwallet/wallet.go +++ /dev/null @@ -1,287 +0,0 @@ -package spvwallet - -import ( - "github.com/elastos/Elastos.ELA.SPV/database" - "github.com/elastos/Elastos.ELA.SPV/sdk" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/config" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/headers" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/sqlite" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" - "github.com/elastos/Elastos.ELA.SPV/util" - - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" -) - -const ( - MaxPeers = 12 - MinPeersForSync = 2 -) - -type Wallet struct { - sdk.IService - rpcServer *rpc.Server - chainStore database.ChainStore - db sqlite.DataStore - filter *sdk.AddrFilter -} - -func (w *Wallet) Start() { - w.IService.Start() - w.rpcServer.Start() -} - -func (w *Wallet) Stop() { - w.IService.Stop() - w.rpcServer.Close() -} - -// Batch returns a TxBatch instance for transactions batch -// commit, this can get better performance when commit a bunch -// of transactions within a block. -func (w *Wallet) Batch() database.TxBatch { - return &txBatch{ - db: w.db, - batch: w.db.Batch(), - filter: w.getAddrFilter(), - } -} - -// HaveTx returns if the transaction already saved in database -// by it's id. -func (w *Wallet) HaveTx(txId *common.Uint256) (bool, error) { - tx, err := w.db.Txs().Get(txId) - return tx != nil, err -} - -// GetTxs returns all transactions within the given height. -func (w *Wallet) GetTxs(height uint32) ([]*util.Tx, error) { - return nil, nil -} - -// RemoveTxs delete all transactions on the given height. Return -// how many transactions are deleted from database. -func (w *Wallet) RemoveTxs(height uint32) (int, error) { - batch := w.db.Batch() - err := batch.RollbackHeight(height) - if err != nil { - return 0, batch.Rollback() - } - return 0, batch.Commit() -} - -// Clear delete all data in database. -func (w *Wallet) Clear() error { - return w.db.Clear() -} - -// Close database. -func (w *Wallet) Close() error { - return w.db.Close() -} - -func (w *Wallet) GetFilterData() ([]*common.Uint168, []*core.OutPoint) { - utxos, err := w.db.UTXOs().GetAll() - if err != nil { - log.Debugf("GetAll UTXOs error: %v", err) - } - stxos, err := w.db.STXOs().GetAll() - if err != nil { - log.Debugf("GetAll STXOs error: %v", err) - } - outpoints := make([]*core.OutPoint, 0, len(utxos)+len(stxos)) - for _, utxo := range utxos { - outpoints = append(outpoints, &utxo.Op) - } - for _, stxo := range stxos { - outpoints = append(outpoints, &stxo.Op) - } - - return w.getAddrFilter().GetAddrs(), outpoints -} - -func (w *Wallet) NotifyNewAddress(hash []byte) { - // Reload address filter to include new address - w.loadAddrFilter() - // Broadcast filterload message to connected peers - w.UpdateFilter() -} - -func (w *Wallet) getAddrFilter() *sdk.AddrFilter { - if w.filter == nil { - w.loadAddrFilter() - } - return w.filter -} - -func (w *Wallet) loadAddrFilter() *sdk.AddrFilter { - addrs, _ := w.db.Addrs().GetAll() - w.filter = sdk.NewAddrFilter(nil) - for _, addr := range addrs { - w.filter.AddAddr(addr.Hash()) - } - return w.filter -} - -// TransactionAnnounce will be invoked when received a new announced transaction. -func (w *Wallet) TransactionAnnounce(tx *core.Transaction) { - // TODO -} - -// TransactionAccepted will be invoked after a transaction sent by -// SendTransaction() method has been accepted. Notice: this method needs at -// lest two connected peers to work. -func (w *Wallet) TransactionAccepted(tx *core.Transaction) { - // TODO -} - -// TransactionRejected will be invoked if a transaction sent by SendTransaction() -// method has been rejected. -func (w *Wallet) TransactionRejected(tx *core.Transaction) { - // TODO -} - -// TransactionConfirmed will be invoked after a transaction sent by -// SendTransaction() method has been packed into a block. -func (w *Wallet) TransactionConfirmed(tx *util.Tx) { - // TODO -} - -// BlockCommitted will be invoked when a block and transactions within it are -// successfully committed into database. -func (w *Wallet) BlockCommitted(block *util.Block) { - if !w.IsCurrent() { - return - } - - w.db.State().PutHeight(block.Height) - // TODO -} - -type txBatch struct { - db sqlite.DataStore - batch sqlite.DataBatch - filter *sdk.AddrFilter -} - -// PutTx add a store transaction operation into batch, and return -// if it is a false positive and error. -func (b *txBatch) PutTx(tx *util.Tx) (bool, error) { - txId := tx.Hash() - height := tx.Height - hits := 0 - - // Check if any UTXOs within this wallet have been spent. - for _, input := range tx.Inputs { - // Move UTXO to STXO - utxo, _ := b.db.UTXOs().Get(&input.Previous) - // Skip if no match. - if utxo == nil { - continue - } - - err := b.batch.STXOs().Put(sutil.NewSTXO(utxo, height, txId)) - if err != nil { - return false, nil - } - hits++ - } - - // Check if there are any output to this wallet address. - for index, output := range tx.Outputs { - // Filter address - if b.filter.ContainAddr(output.ProgramHash) { - var lockTime uint32 - if tx.TxType == core.CoinBase { - lockTime = height + 100 - } - utxo := sutil.NewUTXO(txId, height, index, output.Value, lockTime, output.ProgramHash) - err := b.batch.UTXOs().Put(utxo) - if err != nil { - return false, err - } - hits++ - } - } - - // If no hits, no need to save transaction - if hits == 0 { - return true, nil - } - - // Save transaction - err := b.batch.Txs().Put(sutil.NewTx(tx.Transaction, height)) - if err != nil { - return false, err - } - - return false, nil -} - -// DelTx add a delete transaction operation into batch. -func (b *txBatch) DelTx(txId *common.Uint256) error { - return b.batch.Txs().Del(txId) -} - -// DelTxs add a delete transactions on given height operation. -func (b *txBatch) DelTxs(height uint32) error { - // Delete transactions is used when blockchain doing rollback, this not - // only delete the transactions on the given height, and also restore - // STXOs and remove UTXOs within these transactions. - return b.batch.RollbackHeight(height) -} - -// Rollback cancel all operations in current batch. -func (b *txBatch) Rollback() error { - return b.batch.Rollback() -} - -// Commit the added transactions into database. -func (b *txBatch) Commit() error { - return b.batch.Commit() -} - -func New() (*Wallet, error) { - wallet := new(Wallet) - - // Initialize headers db - headers, err := headers.New() - if err != nil { - return nil, err - } - - // Initialize singleton database - wallet.db, err = sqlite.New() - if err != nil { - return nil, err - } - - // Initiate ChainStore - wallet.chainStore = database.NewDefaultChainDB(headers, wallet) - - // Initialize spv service - wallet.IService, err = sdk.NewService( - &sdk.Config{ - Magic: config.Values().Magic, - SeedList: config.Values().SeedList, - DefaultPort: config.Values().DefaultPort, - MaxPeers: MaxPeers, - MinPeersForSync: MinPeersForSync, - Foundation: config.Values().Foundation, - ChainStore: wallet.chainStore, - GetFilterData: wallet.GetFilterData, - StateNotifier: wallet, - }) - if err != nil { - return nil, err - } - - // Initialize RPC server - server := rpc.InitServer() - server.NotifyNewAddress = wallet.NotifyNewAddress - server.SendTransaction = wallet.IService.SendTransaction - wallet.rpcServer = server - - return wallet, nil -} diff --git a/sync/config.go b/sync/config.go index 9bd4f4b..57b5f6f 100644 --- a/sync/config.go +++ b/sync/config.go @@ -2,9 +2,8 @@ package sync import ( "github.com/elastos/Elastos.ELA.SPV/blockchain" - - "github.com/elastos/Elastos.ELA/bloom" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA.SPV/bloom" + "github.com/elastos/Elastos.ELA.SPV/util" ) const ( @@ -20,7 +19,7 @@ type Config struct { MaxPeers int UpdateFilter func() *bloom.Filter - TransactionAnnounce func(tx *core.Transaction) + TransactionAnnounce func(tx util.Transaction) } func NewDefaultConfig(chain *blockchain.BlockChain, diff --git a/sync/manager.go b/sync/manager.go index a4519d2..5a32f77 100644 --- a/sync/manager.go +++ b/sync/manager.go @@ -10,7 +10,6 @@ import ( "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA/core" ) const ( @@ -65,7 +64,7 @@ type blockMsg struct { // txMsg packages a bitcoin tx message and the peer it came from together // so the block handler has access to that information. type txMsg struct { - tx *core.Transaction + tx util.Transaction peer *peer.Peer reply chan struct{} } @@ -177,16 +176,9 @@ func (sm *SyncManager) startSync() { continue } - // Pick the first available candidate. - if bestPeer == nil { - bestPeer = peer - continue - } - - // Pick the highest available candidate. - if peer.Height() > bestPeer.Height() { - bestPeer = peer - } + // Just pick the first available candidate. + bestPeer = peer + break } // Start syncing from the best peer if one was selected. @@ -677,7 +669,7 @@ func (sm *SyncManager) NewPeer(peer *peer.Peer) { // QueueTx adds the passed transaction message and peer to the block handling // queue. Responds to the done channel argument after the tx message is // processed. -func (sm *SyncManager) QueueTx(tx *core.Transaction, peer *peer.Peer, done chan struct{}) { +func (sm *SyncManager) QueueTx(tx util.Transaction, peer *peer.Peer, done chan struct{}) { // Don't accept more transactions if we're shutting down. if atomic.LoadInt32(&sm.shutdown) != 0 { done <- struct{}{} diff --git a/util/block.go b/util/block.go index df9eb53..8d17490 100644 --- a/util/block.go +++ b/util/block.go @@ -1,7 +1,5 @@ package util -import "github.com/elastos/Elastos.ELA/core" - // Block represent a block that stored in the // blockchain database. type Block struct { @@ -9,5 +7,5 @@ type Block struct { Header // Transactions of this block. - Transactions []*core.Transaction + Transactions []Transaction } diff --git a/util/header.go b/util/header.go index 3f91bd2..22d3e25 100644 --- a/util/header.go +++ b/util/header.go @@ -7,13 +7,14 @@ import ( "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA/core" ) // Header is a data structure stored in database. type Header struct { // The origin header of the block - core.Header + BlockHeader + + Height uint32 // MerkleProof for transactions packed in this block NumTxs uint32 @@ -27,7 +28,12 @@ type Header struct { func (h *Header) Serialize() ([]byte, error) { buf := new(bytes.Buffer) - err := h.Header.Serialize(buf) + err := h.BlockHeader.Serialize(buf) + if err != nil { + return nil, err + } + + err = common.WriteUint32(buf, h.Height) if err != nil { return nil, err } @@ -62,7 +68,12 @@ func (h *Header) Serialize() ([]byte, error) { func (h *Header) Deserialize(b []byte) error { r := bytes.NewReader(b) - err := h.Header.Deserialize(r) + err := h.BlockHeader.Deserialize(r) + if err != nil { + return err + } + + h.Height, err = common.ReadUint32(r) if err != nil { return err } diff --git a/util/interface.go b/util/interface.go new file mode 100644 index 0000000..f8b9790 --- /dev/null +++ b/util/interface.go @@ -0,0 +1,23 @@ +package util + +import ( + "io" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +type BlockHeader interface { + Previous() common.Uint256 + Bits() uint32 + MerkleRoot() common.Uint256 + Hash() common.Uint256 + PowHash() common.Uint256 + Serialize(w io.Writer) error + Deserialize(r io.Reader) error +} + +type Transaction interface { + Hash() common.Uint256 + Serialize(w io.Writer) error + Deserialize(r io.Reader) error +} \ No newline at end of file diff --git a/util/outpoint.go b/util/outpoint.go new file mode 100644 index 0000000..c029638 --- /dev/null +++ b/util/outpoint.go @@ -0,0 +1,54 @@ +package util + +import ( + "bytes" + "io" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +type OutPoint struct { + TxID common.Uint256 + Index uint16 +} + +func (op *OutPoint) IsEqual(o OutPoint) bool { + if !op.TxID.IsEqual(o.TxID) { + return false + } + if op.Index != o.Index { + return false + } + return true +} + +func (op *OutPoint) Serialize(w io.Writer) error { + return common.WriteElements(w, &op.TxID, op.Index) +} + +func (op *OutPoint) Deserialize(r io.Reader) error { + return common.ReadElements(r, &op.TxID, &op.Index) +} + +func (op *OutPoint) Bytes() []byte { + buf := new(bytes.Buffer) + op.Serialize(buf) + return buf.Bytes() +} + +func NewOutPoint(txId common.Uint256, index uint16) *OutPoint { + return &OutPoint{ + TxID: txId, + Index: index, + } +} + +func OutPointFromBytes(value []byte) (*OutPoint, error) { + outPoint := new(OutPoint) + err := outPoint.Deserialize(bytes.NewReader(value)) + if err != nil { + return nil, err + } + + return outPoint, nil +} diff --git a/util/tx.go b/util/tx.go index e959753..a021fca 100644 --- a/util/tx.go +++ b/util/tx.go @@ -1,39 +1,76 @@ package util import ( - "encoding/binary" + "bytes" "io" + "time" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" ) // Tx is a data structure used in database. type Tx struct { - // The origin transaction data. - core.Transaction + // The transaction hash. + Hash common.Uint256 // The block height that this transaction // belongs to. Height uint32 + + // The time the transaction was first seen + Timestamp time.Time + + // Transaction + RawData []byte } -func NewTx(tx core.Transaction, height uint32) *Tx { +func NewTx(tx Transaction, height uint32) *Tx { + buf := new(bytes.Buffer) + tx.Serialize(buf) return &Tx{ - Transaction: tx, - Height: height, + Hash: tx.Hash(), + Height: height, + Timestamp: time.Now(), + RawData: buf.Bytes(), } } -func (t *Tx) Serialize(buf io.Writer) error { - if err := t.Transaction.Serialize(buf); err != nil { +func (t *Tx) Serialize(w io.Writer) error { + if err := t.Hash.Serialize(w); err != nil { + return err + } + + if err := common.WriteUint32(w, t.Height); err != nil { + return err + } + + err := common.WriteUint64(w, uint64(t.Timestamp.Unix())) + if err != nil { return err } - return binary.Write(buf, binary.LittleEndian, t.Height) + + return common.WriteVarBytes(w, t.RawData) } -func (t *Tx) Deserialize(reader io.Reader) error { - if err := t.Transaction.Deserialize(reader); err != nil { +func (t *Tx) Deserialize(r io.Reader) error { + if err := t.Hash.Deserialize(r); err != nil { + return err + } + + var err error + t.Height, err = common.ReadUint32(r) + if err != nil { return err } - return binary.Read(reader, binary.LittleEndian, &t.Height) + + timestamp, err := common.ReadUint64(r) + if err != nil { + return err + } + t.Timestamp = time.Unix(int64(timestamp), 0) + + t.RawData, err = common.ReadVarBytes(r, msg.MaxBlockSize, + "Tx RawData") + return err } diff --git a/wallet/client.go b/wallet/client.go new file mode 100644 index 0000000..5556eb4 --- /dev/null +++ b/wallet/client.go @@ -0,0 +1,37 @@ +package wallet + +import ( + "github.com/elastos/Elastos.ELA.SPV/util" + "os" + + "github.com/elastos/Elastos.ELA.SPV/wallet/client" + "github.com/elastos/Elastos.ELA.SPV/wallet/client/account" + "github.com/elastos/Elastos.ELA.SPV/wallet/client/transaction" + "github.com/elastos/Elastos.ELA.SPV/wallet/client/wallet" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/urfave/cli" +) + +func RunClient(version, rpcUrl string, assetId common.Uint256, newBlockHeader func() util.BlockHeader) { + client.Setup(rpcUrl, assetId, newBlockHeader) + + app := cli.NewApp() + app.Name = "ELASTOS SPV WALLET" + app.Version = version + app.HelpName = "ELASTOS SPV WALLET HELP" + app.Usage = "command line user interface" + app.UsageText = "[global option] command [command options] [args]" + app.HideHelp = false + app.HideVersion = false + //commands + app.Commands = []cli.Command{ + wallet.NewCreateCommand(), + wallet.NewChangePasswordCommand(), + wallet.NewResetCommand(), + account.NewCommand(), + transaction.NewCommand(), + } + + app.Run(os.Args) +} diff --git a/spvwallet/client/account/account.go b/wallet/client/account/account.go similarity index 95% rename from spvwallet/client/account/account.go rename to wallet/client/account/account.go index 44ee0aa..854ac1b 100644 --- a/spvwallet/client/account/account.go +++ b/wallet/client/account/account.go @@ -7,7 +7,7 @@ import ( "os" "strings" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/client" + "github.com/elastos/Elastos.ELA.SPV/wallet/client" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" @@ -18,7 +18,7 @@ const ( MinMultiSignKeys = 3 ) -func listBalanceInfo(wallet client.Wallet) error { +func listBalanceInfo(wallet *client.Wallet) error { addrs, err := wallet.GetAddrs() if err != nil { return errors.New("get wallet addresses failed") @@ -27,7 +27,7 @@ func listBalanceInfo(wallet client.Wallet) error { return client.ShowAccounts(addrs, nil, wallet) } -func newSubAccount(password []byte, wallet client.Wallet) error { +func newSubAccount(password []byte, wallet *client.Wallet) error { var err error password, err = client.GetPassword(password, false) if err != nil { @@ -47,7 +47,7 @@ func newSubAccount(password []byte, wallet client.Wallet) error { return client.ShowAccounts(addrs, programHash, wallet) } -func addMultiSignAccount(context *cli.Context, wallet client.Wallet, content string) error { +func addMultiSignAccount(context *cli.Context, wallet *client.Wallet, content string) error { // Get address content from file or cli input publicKeys, err := getPublicKeys(content) if err != nil { diff --git a/spvwallet/client/common.go b/wallet/client/common.go similarity index 96% rename from spvwallet/client/common.go rename to wallet/client/common.go index c33faa8..2baa163 100644 --- a/spvwallet/client/common.go +++ b/wallet/client/common.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/howeyc/gopass" @@ -86,7 +86,7 @@ func ShowAccountInfo(password []byte) error { return nil } -func SelectAccount(wallet Wallet) (string, error) { +func SelectAccount(wallet *Wallet) (string, error) { addrs, err := wallet.GetAddrs() if err != nil || len(addrs) == 0 { return "", errors.New("fail to load wallet addresses") @@ -114,7 +114,7 @@ func SelectAccount(wallet Wallet) (string, error) { return addrs[index].String(), nil } -func ShowAccounts(addrs []*sutil.Addr, newAddr *common.Uint168, wallet Wallet) error { +func ShowAccounts(addrs []*sutil.Addr, newAddr *common.Uint168, wallet *Wallet) error { // print header fmt.Printf("%5s %34s %-20s%22s %6s\n", "INDEX", "ADDRESS", "BALANCE", "(LOCKED)", "TYPE") fmt.Println("-----", strings.Repeat("-", 34), strings.Repeat("-", 42), "------") diff --git a/spvwallet/client/database/database.go b/wallet/client/database/database.go similarity index 66% rename from spvwallet/client/database/database.go rename to wallet/client/database/database.go index 205843a..47090a1 100644 --- a/spvwallet/client/database/database.go +++ b/wallet/client/database/database.go @@ -3,39 +3,31 @@ package database import ( "sync" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/headers" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/sqlite" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.SPV/wallet/store/headers" + "github.com/elastos/Elastos.ELA.SPV/wallet/store/sqlite" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" ) -type Database interface { - AddAddress(address *common.Uint168, script []byte, addrType int) error - GetAddress(address *common.Uint168) (*sutil.Addr, error) - GetAddrs() ([]*sutil.Addr, error) - DeleteAddress(address *common.Uint168) error - GetAddressUTXOs(address *common.Uint168) ([]*sutil.UTXO, error) - GetAddressSTXOs(address *common.Uint168) ([]*sutil.STXO, error) - BestHeight() uint32 - Clear() error -} - -func New() (Database, error) { - dataStore, err := sqlite.New() +func New(newBlockHeader func() util.BlockHeader) (*database, error) { + dataStore, err := sqlite.NewDatabase() if err != nil { return nil, err } return &database{ - lock: new(sync.RWMutex), - store: dataStore, + lock: new(sync.RWMutex), + store: dataStore, + newBlockHeader: newBlockHeader, }, nil } type database struct { - lock *sync.RWMutex - store sqlite.DataStore + lock *sync.RWMutex + store sqlite.DataStore + newBlockHeader func() util.BlockHeader } func (d *database) AddAddress(address *common.Uint168, script []byte, addrType int) error { @@ -91,7 +83,7 @@ func (d *database) Clear() error { d.lock.Lock() defer d.lock.Unlock() - headers, err := headers.New() + headers, err := headers.NewDatabase(d.newBlockHeader) if err != nil { return err } diff --git a/wallet/client/database/interface.go b/wallet/client/database/interface.go new file mode 100644 index 0000000..48f6a00 --- /dev/null +++ b/wallet/client/database/interface.go @@ -0,0 +1,18 @@ +package database + +import ( + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +type Database interface { + AddAddress(address *common.Uint168, script []byte, addrType int) error + GetAddress(address *common.Uint168) (*sutil.Addr, error) + GetAddrs() ([]*sutil.Addr, error) + DeleteAddress(address *common.Uint168) error + GetAddressUTXOs(address *common.Uint168) ([]*sutil.UTXO, error) + GetAddressSTXOs(address *common.Uint168) ([]*sutil.STXO, error) + BestHeight() uint32 + Clear() error +} diff --git a/spvwallet/client/keystore.go b/wallet/client/keystore.go similarity index 78% rename from spvwallet/client/keystore.go rename to wallet/client/keystore.go index bc4ef9a..7539090 100644 --- a/spvwallet/client/keystore.go +++ b/wallet/client/keystore.go @@ -16,20 +16,7 @@ const ( KeystoreVersion = "1.0" ) -type Keystore interface { - ChangePassword(old, new []byte) error - - MainAccount() *sdk.Account - NewAccount() *sdk.Account - GetAccounts() []*sdk.Account - GetAccountByIndex(index int) *sdk.Account - GetAccountByProgramHash(programHash *common.Uint168) *sdk.Account - - Json() (string, error) - FromJson(json string, password string) error -} - -type KeystoreImpl struct { +type Keystore struct { sync.Mutex *KeystoreFile @@ -39,13 +26,13 @@ type KeystoreImpl struct { accounts []*sdk.Account } -func CreateKeystore(password []byte) (Keystore, error) { +func CreateKeystore(password []byte) (*Keystore, error) { keystoreFile, err := CreateKeystoreFile() if err != nil { return nil, err } - keystore := &KeystoreImpl{ + keystore := &Keystore{ KeystoreFile: keystoreFile, } @@ -99,17 +86,17 @@ func CreateKeystore(password []byte) (Keystore, error) { return keystore, nil } -func OpenKeystore(password []byte) (Keystore, error) { +func OpenKeystore(password []byte) (*Keystore, error) { keystoreFile, err := OpenKeystoreFile() if err != nil { return nil, err } - keystore := new(KeystoreImpl) + keystore := new(Keystore) err = keystore.initKeystore(keystoreFile, password) return keystore, err } -func (store *KeystoreImpl) initKeystore(keystoreFile *KeystoreFile, password []byte) error { +func (store *Keystore) initKeystore(keystoreFile *KeystoreFile, password []byte) error { store.KeystoreFile = keystoreFile err := store.verifyPassword(password) if err != nil { @@ -128,7 +115,7 @@ func (store *KeystoreImpl) initKeystore(keystoreFile *KeystoreFile, password []b return store.initAccounts(masterKey, privateKey, publicKey) } -func (store *KeystoreImpl) initAccounts(masterKey, privateKey []byte, publicKey *crypto.PublicKey) error { +func (store *Keystore) initAccounts(masterKey, privateKey []byte, publicKey *crypto.PublicKey) error { // initiate main account mainAccount, err := sdk.NewAccount(privateKey, publicKey) if err != nil { @@ -157,7 +144,7 @@ func (store *KeystoreImpl) initAccounts(masterKey, privateKey []byte, publicKey return nil } -func (store *KeystoreImpl) verifyPassword(password []byte) error { +func (store *Keystore) verifyPassword(password []byte) error { passwordKey := crypto.ToAesKey(password) passwordHash := sha256.Sum256(passwordKey) @@ -171,7 +158,7 @@ func (store *KeystoreImpl) verifyPassword(password []byte) error { return errors.New("password wrong") } -func (store *KeystoreImpl) ChangePassword(oldPassword, newPassword []byte) error { +func (store *Keystore) ChangePassword(oldPassword, newPassword []byte) error { // Get old passwordKey oldPasswordKey := crypto.ToAesKey(oldPassword) @@ -218,11 +205,11 @@ func (store *KeystoreImpl) ChangePassword(oldPassword, newPassword []byte) error return nil } -func (store *KeystoreImpl) MainAccount() *sdk.Account { +func (store *Keystore) MainAccount() *sdk.Account { return store.GetAccountByIndex(0) } -func (store *KeystoreImpl) NewAccount() *sdk.Account { +func (store *Keystore) NewAccount() *sdk.Account { // create sub account privateKey, publicKey, err := crypto.GenerateSubKeyPair( store.SubAccountsCount+1, store.masterKey, store.accounts[0].PrivateKey()) @@ -246,18 +233,18 @@ func (store *KeystoreImpl) NewAccount() *sdk.Account { return account } -func (store *KeystoreImpl) GetAccounts() []*sdk.Account { +func (store *Keystore) GetAccounts() []*sdk.Account { return store.accounts } -func (store *KeystoreImpl) GetAccountByIndex(index int) *sdk.Account { +func (store *Keystore) GetAccountByIndex(index int) *sdk.Account { if index < 0 || index > len(store.accounts)-1 { return nil } return store.accounts[index] } -func (store *KeystoreImpl) GetAccountByProgramHash(programHash *common.Uint168) *sdk.Account { +func (store *Keystore) GetAccountByProgramHash(programHash *common.Uint168) *sdk.Account { if programHash == nil { return nil } @@ -269,7 +256,7 @@ func (store *KeystoreImpl) GetAccountByProgramHash(programHash *common.Uint168) return nil } -func (store *KeystoreImpl) encryptMasterKey(passwordKey, masterKey []byte) ([]byte, error) { +func (store *Keystore) encryptMasterKey(passwordKey, masterKey []byte) ([]byte, error) { iv, err := store.GetIV() if err != nil { return nil, err @@ -283,7 +270,7 @@ func (store *KeystoreImpl) encryptMasterKey(passwordKey, masterKey []byte) ([]by return masterKeyEncrypted, nil } -func (store *KeystoreImpl) decryptMasterKey(passwordKey []byte) (masterKey []byte, err error) { +func (store *Keystore) decryptMasterKey(passwordKey []byte) (masterKey []byte, err error) { iv, err := store.GetIV() if err != nil { return nil, err @@ -302,7 +289,7 @@ func (store *KeystoreImpl) decryptMasterKey(passwordKey []byte) (masterKey []byt return masterKey, nil } -func (store *KeystoreImpl) encryptPrivateKey(masterKey, passwordKey, privateKey []byte, publicKey *crypto.PublicKey) ([]byte, error) { +func (store *Keystore) encryptPrivateKey(masterKey, passwordKey, privateKey []byte, publicKey *crypto.PublicKey) ([]byte, error) { decryptedPrivateKey := make([]byte, 96) publicKeyBytes, err := publicKey.EncodePoint(false) @@ -328,7 +315,7 @@ func (store *KeystoreImpl) encryptPrivateKey(masterKey, passwordKey, privateKey return encryptedPrivateKey, nil } -func (store *KeystoreImpl) decryptPrivateKey(masterKey, passwordKey []byte) ([]byte, *crypto.PublicKey, error) { +func (store *Keystore) decryptPrivateKey(masterKey, passwordKey []byte) ([]byte, *crypto.PublicKey, error) { privateKeyEncrypted, err := store.GetPrivetKeyEncrypted() if err != nil { return nil, nil, err @@ -351,12 +338,12 @@ func (store *KeystoreImpl) decryptPrivateKey(masterKey, passwordKey []byte) ([]b return privateKey, crypto.NewPubKey(privateKey), nil } -func (store *KeystoreImpl) FromJson(str string, password string) error { +func (store *Keystore) FromJson(str string, password string) error { file := new(KeystoreFile) file.FromJson(str) return store.initKeystore(file, []byte(password)) } -func (store *KeystoreImpl) Json() (string, error) { +func (store *Keystore) Json() (string, error) { return store.KeystoreFile.Json() } diff --git a/spvwallet/client/keystore_file.go b/wallet/client/keystore_file.go similarity index 100% rename from spvwallet/client/keystore_file.go rename to wallet/client/keystore_file.go diff --git a/spvwallet/client/transaction/transaction.go b/wallet/client/transaction/transaction.go similarity index 85% rename from spvwallet/client/transaction/transaction.go rename to wallet/client/transaction/transaction.go index d028c7e..05ff3ee 100644 --- a/spvwallet/client/transaction/transaction.go +++ b/wallet/client/transaction/transaction.go @@ -10,7 +10,7 @@ import ( "strconv" "strings" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/client" + "github.com/elastos/Elastos.ELA.SPV/wallet/client" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" @@ -18,7 +18,7 @@ import ( "github.com/urfave/cli" ) -func CreateTransaction(c *cli.Context, wallet client.Wallet) error { +func CreateTransaction(c *cli.Context, wallet *client.Wallet) error { txn, err := createTransaction(c, wallet) if err != nil { return err @@ -26,7 +26,7 @@ func CreateTransaction(c *cli.Context, wallet client.Wallet) error { return output(txn) } -func createTransaction(c *cli.Context, wallet client.Wallet) (*core.Transaction, error) { +func createTransaction(c *cli.Context, wallet *client.Wallet) (*core.Transaction, error) { feeStr := c.String("fee") if feeStr == "" { return nil, errors.New("use --fee to specify transfer fee") @@ -45,15 +45,15 @@ func createTransaction(c *cli.Context, wallet client.Wallet) (*core.Transaction, } } - var txn *core.Transaction + var tx *core.Transaction multiOutput := c.String("file") if multiOutput != "" { - txn, err = createMultiOutputTransaction(c, wallet, multiOutput, from, fee) + tx, err = createMultiOutputTransaction(c, wallet, multiOutput, from, fee) if err != nil { return nil, err } - return txn, nil + return tx, nil } to := c.String("to") @@ -73,7 +73,7 @@ func createTransaction(c *cli.Context, wallet client.Wallet) (*core.Transaction, lockStr := c.String("lock") if lockStr == "" { - txn, err = wallet.CreateTransaction(from, to, amount, fee) + tx, err = wallet.CreateTransaction(from, to, amount, fee) if err != nil { return nil, errors.New("create transaction failed: " + err.Error()) } @@ -82,16 +82,16 @@ func createTransaction(c *cli.Context, wallet client.Wallet) (*core.Transaction, if err != nil { return nil, errors.New("invalid lock height") } - txn, err = wallet.CreateLockedTransaction(from, to, amount, fee, uint32(lock)) + tx, err = wallet.CreateLockedTransaction(from, to, amount, fee, uint32(lock)) if err != nil { return nil, errors.New("create transaction failed: " + err.Error()) } } - return txn, nil + return tx, nil } -func createMultiOutputTransaction(c *cli.Context, wallet client.Wallet, path, from string, fee *common.Fixed64) (*core.Transaction, error) { +func createMultiOutputTransaction(c *cli.Context, wallet *client.Wallet, path, from string, fee *common.Fixed64) (*core.Transaction, error) { if _, err := os.Stat(path); err != nil { return nil, errors.New("invalid multi output file path") } @@ -117,9 +117,9 @@ func createMultiOutputTransaction(c *cli.Context, wallet client.Wallet, path, fr } lockStr := c.String("lock") - var txn *core.Transaction + var tx *core.Transaction if lockStr == "" { - txn, err = wallet.CreateMultiOutputTransaction(from, fee, multiOutput...) + tx, err = wallet.CreateMultiOutputTransaction(from, fee, multiOutput...) if err != nil { return nil, errors.New("create multi output transaction failed: " + err.Error()) } @@ -128,16 +128,16 @@ func createMultiOutputTransaction(c *cli.Context, wallet client.Wallet, path, fr if err != nil { return nil, errors.New("invalid lock height") } - txn, err = wallet.CreateLockedMultiOutputTransaction(from, fee, uint32(lock), multiOutput...) + tx, err = wallet.CreateLockedMultiOutputTransaction(from, fee, uint32(lock), multiOutput...) if err != nil { return nil, errors.New("create multi output transaction failed: " + err.Error()) } } - return txn, nil + return tx, nil } -func SignTransaction(password []byte, context *cli.Context, wallet client.Wallet) error { +func SignTransaction(password []byte, context *cli.Context, wallet *client.Wallet) error { txn, err := getTransaction(context) if err != nil { return err @@ -151,8 +151,8 @@ func SignTransaction(password []byte, context *cli.Context, wallet client.Wallet return output(txn) } -func signTransaction(password []byte, wallet client.Wallet, txn *core.Transaction) (*core.Transaction, error) { - haveSign, needSign, err := crypto.GetSignStatus(txn.Programs[0].Code, txn.Programs[0].Parameter) +func signTransaction(password []byte, wallet *client.Wallet, tx *core.Transaction) (*core.Transaction, error) { + haveSign, needSign, err := crypto.GetSignStatus(tx.Programs[0].Code, tx.Programs[0].Parameter) if haveSign == needSign { return nil, errors.New("transaction was fully signed, no need more sign") } @@ -162,43 +162,43 @@ func signTransaction(password []byte, wallet client.Wallet, txn *core.Transactio return nil, err } - return wallet.Sign(password, txn) + return wallet.Sign(password, tx) } -func SendTransaction(password []byte, context *cli.Context, wallet client.Wallet) error { +func SendTransaction(password []byte, context *cli.Context, wallet *client.Wallet) error { content, err := getContent(context) - var txn *core.Transaction + var tx *core.Transaction if content == nil { // Create transaction with command line arguments - txn, err = createTransaction(context, wallet) + tx, err = createTransaction(context, wallet) if err != nil { return err } // Sign transaction - txn, err = signTransaction(password, wallet, txn) + tx, err = signTransaction(password, wallet, tx) if err != nil { return err } } else { - txn = new(core.Transaction) + tx = new(core.Transaction) data, err := common.HexStringToBytes(*content) if err != nil { return fmt.Errorf("Deseralize transaction file failed, error %s", err.Error()) } - err = txn.Deserialize(bytes.NewReader(data)) + err = tx.Deserialize(bytes.NewReader(data)) if err != nil { return err } } - err = wallet.SendTransaction(txn) + err = wallet.SendTransaction(tx) if err != nil { return err } // Return reversed hex string - fmt.Println(common.BytesToHexString(txn.Hash().Bytes())) + fmt.Println(common.BytesToHexString(tx.Hash().Bytes())) return nil } @@ -254,10 +254,10 @@ func getTransaction(context *cli.Context) (*core.Transaction, error) { return &txn, nil } -func output(txn *core.Transaction) error { +func output(tx *core.Transaction) error { // Serialise transaction content buf := new(bytes.Buffer) - txn.Serialize(buf) + tx.Serialize(buf) content := common.BytesToHexString(buf.Bytes()) // Print transaction hex string content to console @@ -266,14 +266,14 @@ func output(txn *core.Transaction) error { // Output to file fileName := "to_be_signed" // Create transaction file name - haveSign, needSign, _ := crypto.GetSignStatus(txn.Programs[0].Code, txn.Programs[0].Parameter) + haveSign, needSign, _ := crypto.GetSignStatus(tx.Programs[0].Code, tx.Programs[0].Parameter) if needSign > haveSign { fileName = fmt.Sprint(fileName, "_", haveSign, "_of_", needSign) } else if needSign == haveSign { fileName = "ready_to_send" } - fileName = fileName + ".txn" + fileName = fileName + ".tx" file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) if err != nil { diff --git a/spvwallet/client/interface.go b/wallet/client/wallet.go similarity index 63% rename from spvwallet/client/interface.go rename to wallet/client/wallet.go index eaa8b14..01b73e7 100644 --- a/spvwallet/client/interface.go +++ b/wallet/client/wallet.go @@ -2,48 +2,44 @@ package client import ( "bytes" + "encoding/hex" "errors" + util2 "github.com/elastos/Elastos.ELA.SPV/util" "math" "math/rand" "strconv" "github.com/elastos/Elastos.ELA.SPV/sdk" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/client/database" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/rpc" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.SPV/wallet/client/database" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" + "github.com/elastos/Elastos.ELA.Utility/http/jsonrpc" + "github.com/elastos/Elastos.ELA.Utility/http/util" "github.com/elastos/Elastos.ELA/core" ) -var SystemAssetId = getSystemAssetId() +var ( + jsonRpcUrl string + sysAssetId common.Uint256 + newHeader func() util2.BlockHeader +) type Transfer struct { Address string Value *common.Fixed64 } -type Wallet interface { +type Wallet struct { database.Database - - VerifyPassword(password []byte) error - ChangePassword(oldPassword, newPassword []byte) error - - NewSubAccount(password []byte) (*common.Uint168, error) - AddMultiSignAccount(M uint, publicKey ...*crypto.PublicKey) (*common.Uint168, error) - - CreateTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64) (*core.Transaction, error) - CreateLockedTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64, lockedUntil uint32) (*core.Transaction, error) - CreateMultiOutputTransaction(fromAddress string, fee *common.Fixed64, output ...*Transfer) (*core.Transaction, error) - CreateLockedMultiOutputTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, output ...*Transfer) (*core.Transaction, error) - Sign(password []byte, transaction *core.Transaction) (*core.Transaction, error) - SendTransaction(txn *core.Transaction) error + *Keystore } -type wallet struct { - database.Database - Keystore +func Setup(rpcUrl string, assetId common.Uint256, newBlockHeader func() util2.BlockHeader) { + jsonRpcUrl = rpcUrl + sysAssetId = assetId + newHeader = newBlockHeader } func Create(password []byte) error { @@ -52,28 +48,28 @@ func Create(password []byte) error { return err } - database, err := database.New() + db, err := database.New(newHeader) if err != nil { return err } mainAccount := keyStore.GetAccountByIndex(0) - return database.AddAddress(mainAccount.ProgramHash(), + return db.AddAddress(mainAccount.ProgramHash(), mainAccount.RedeemScript(), sutil.TypeMaster) } -func Open() (Wallet, error) { - database, err := database.New() +func Open() (*Wallet, error) { + db, err := database.New(newHeader) if err != nil { return nil, err } - return &wallet{ - Database: database, + return &Wallet{ + Database: db, }, nil } -func (wallet *wallet) VerifyPassword(password []byte) error { +func (wallet *Wallet) VerifyPassword(password []byte) error { keyStore, err := OpenKeystore(password) if err != nil { return err @@ -82,7 +78,7 @@ func (wallet *wallet) VerifyPassword(password []byte) error { return nil } -func (wallet *wallet) NewSubAccount(password []byte) (*common.Uint168, error) { +func (wallet *Wallet) NewSubAccount(password []byte) (*common.Uint168, error) { err := wallet.VerifyPassword(password) if err != nil { return nil, err @@ -95,12 +91,12 @@ func (wallet *wallet) NewSubAccount(password []byte) (*common.Uint168, error) { } // Notify SPV service to reload bloom filter with the new address - rpc.GetClient().NotifyNewAddress(account.ProgramHash().Bytes()) + jsonrpc.Call(jsonRpcUrl, util.Params{"addr": account.ProgramHash().String()}) return account.ProgramHash(), nil } -func (wallet *wallet) AddMultiSignAccount(M uint, publicKeys ...*crypto.PublicKey) (*common.Uint168, error) { +func (wallet *Wallet) AddMultiSignAccount(M uint, publicKeys ...*crypto.PublicKey) (*common.Uint168, error) { redeemScript, err := crypto.CreateMultiSignRedeemScript(M, publicKeys) if err != nil { return nil, errors.New("[Wallet], CreateStandardRedeemScript failed") @@ -117,28 +113,34 @@ func (wallet *wallet) AddMultiSignAccount(M uint, publicKeys ...*crypto.PublicKe } // Notify SPV service to reload bloom filter with the new address - rpc.GetClient().NotifyNewAddress(programHash.Bytes()) + jsonrpc.Call(jsonRpcUrl, util.Params{"addr": programHash.String()}) return programHash, nil } -func (wallet *wallet) CreateTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64) (*core.Transaction, error) { +func (wallet *Wallet) CreateTransaction(fromAddress, toAddress string, amount, + fee *common.Fixed64) (*core.Transaction, error) { return wallet.CreateLockedTransaction(fromAddress, toAddress, amount, fee, uint32(0)) } -func (wallet *wallet) CreateLockedTransaction(fromAddress, toAddress string, amount, fee *common.Fixed64, lockedUntil uint32) (*core.Transaction, error) { - return wallet.CreateLockedMultiOutputTransaction(fromAddress, fee, lockedUntil, &Transfer{toAddress, amount}) +func (wallet *Wallet) CreateLockedTransaction(fromAddress, toAddress string, + amount, fee *common.Fixed64, lockedUntil uint32) (*core.Transaction, error) { + return wallet.CreateLockedMultiOutputTransaction(fromAddress, fee, lockedUntil, + &Transfer{toAddress, amount}) } -func (wallet *wallet) CreateMultiOutputTransaction(fromAddress string, fee *common.Fixed64, outputs ...*Transfer) (*core.Transaction, error) { +func (wallet *Wallet) CreateMultiOutputTransaction(fromAddress string, fee *common.Fixed64, + outputs ...*Transfer) (*core.Transaction, error) { return wallet.CreateLockedMultiOutputTransaction(fromAddress, fee, uint32(0), outputs...) } -func (wallet *wallet) CreateLockedMultiOutputTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, outputs ...*Transfer) (*core.Transaction, error) { +func (wallet *Wallet) CreateLockedMultiOutputTransaction(fromAddress string, fee *common.Fixed64, + lockedUntil uint32, outputs ...*Transfer) (*core.Transaction, error) { return wallet.createTransaction(fromAddress, fee, lockedUntil, outputs...) } -func (wallet *wallet) createTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, outputs ...*Transfer) (*core.Transaction, error) { +func (wallet *Wallet) createTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, + outputs ...*Transfer) (*core.Transaction, error) { // Check if output is valid if outputs == nil || len(outputs) == 0 { return nil, errors.New("[Wallet], Invalid transaction target") @@ -160,7 +162,7 @@ func (wallet *wallet) createTransaction(fromAddress string, fee *common.Fixed64, return nil, errors.New("[Wallet], Invalid receiver address") } txOutput := &core.Output{ - AssetID: SystemAssetId, + AssetID: sysAssetId, ProgramHash: *receiver, Value: *output.Value, OutputLock: lockedUntil, @@ -187,7 +189,7 @@ func (wallet *wallet) createTransaction(fromAddress string, fee *common.Fixed64, break } else if utxo.Value > totalOutputValue { change := &core.Output{ - AssetID: SystemAssetId, + AssetID: sysAssetId, Value: utxo.Value - totalOutputValue, OutputLock: uint32(0), ProgramHash: *spender, @@ -209,14 +211,14 @@ func (wallet *wallet) createTransaction(fromAddress string, fee *common.Fixed64, return wallet.newTransaction(addr.Script(), txInputs, txOutputs), nil } -func (wallet *wallet) Sign(password []byte, txn *core.Transaction) (*core.Transaction, error) { +func (wallet *Wallet) Sign(password []byte, tx *core.Transaction) (*core.Transaction, error) { // Verify password err := wallet.VerifyPassword(password) if err != nil { return nil, err } // Get sign type - signType, err := crypto.GetScriptType(txn.Programs[0].Code) + signType, err := crypto.GetScriptType(tx.Programs[0].Code) if err != nil { return nil, err } @@ -224,7 +226,7 @@ func (wallet *wallet) Sign(password []byte, txn *core.Transaction) (*core.Transa if signType == common.STANDARD { // Sign single transaction - txn, err = wallet.signStandardTransaction(txn) + tx, err = wallet.signStandardTransaction(tx) if err != nil { return nil, err } @@ -232,17 +234,17 @@ func (wallet *wallet) Sign(password []byte, txn *core.Transaction) (*core.Transa } else if signType == common.MULTISIG { // Sign multi sign transaction - txn, err = wallet.signMultiSigTransaction(txn) + tx, err = wallet.signMultiSigTransaction(tx) if err != nil { return nil, err } } - return txn, nil + return tx, nil } -func (wallet *wallet) signStandardTransaction(txn *core.Transaction) (*core.Transaction, error) { - code := txn.Programs[0].Code +func (wallet *Wallet) signStandardTransaction(tx *core.Transaction) (*core.Transaction, error) { + code := tx.Programs[0].Code // Get signer programHash, err := crypto.GetSigner(code) // Check if current user is a valid signer @@ -252,7 +254,7 @@ func (wallet *wallet) signStandardTransaction(txn *core.Transaction) (*core.Tran } // Sign transaction buf := new(bytes.Buffer) - txn.SerializeUnsigned(buf) + tx.SerializeUnsigned(buf) signature, err := account.Sign(buf.Bytes()) if err != nil { return nil, err @@ -263,14 +265,14 @@ func (wallet *wallet) signStandardTransaction(txn *core.Transaction) (*core.Tran buf.Write(signature) // Set program var program = &core.Program{code, buf.Bytes()} - txn.Programs = []*core.Program{program} + tx.Programs = []*core.Program{program} - return txn, nil + return tx, nil } -func (wallet *wallet) signMultiSigTransaction(txn *core.Transaction) (*core.Transaction, error) { - code := txn.Programs[0].Code - param := txn.Programs[0].Parameter +func (wallet *Wallet) signMultiSigTransaction(tx *core.Transaction) (*core.Transaction, error) { + code := tx.Programs[0].Code + param := tx.Programs[0].Parameter // Check if current user is a valid signer var signerIndex = -1 programHashes, err := crypto.GetSigners(code) @@ -290,47 +292,32 @@ func (wallet *wallet) signMultiSigTransaction(txn *core.Transaction) (*core.Tran } // Sign transaction buf := new(bytes.Buffer) - txn.SerializeUnsigned(buf) + tx.SerializeUnsigned(buf) signedTx, err := account.Sign(buf.Bytes()) if err != nil { return nil, err } // Append signature - txn.Programs[0].Parameter, err = crypto.AppendSignature(signerIndex, signedTx, buf.Bytes(), code, param) + tx.Programs[0].Parameter, err = crypto.AppendSignature(signerIndex, signedTx, buf.Bytes(), code, param) if err != nil { return nil, err } - return txn, nil + return tx, nil } -func (wallet *wallet) SendTransaction(txn *core.Transaction) error { - // Send transaction through P2P network - return rpc.GetClient().SendTransaction(txn) -} - -func getSystemAssetId() common.Uint256 { - systemToken := &core.Transaction{ - TxType: core.RegisterAsset, - PayloadVersion: 0, - Payload: &core.PayloadRegisterAsset{ - Asset: core.Asset{ - Name: "ELA", - Precision: 0x08, - AssetType: 0x00, - }, - Amount: 0 * 100000000, - Controller: common.Uint168{}, - }, - Attributes: []*core.Attribute{}, - Inputs: []*core.Input{}, - Outputs: []*core.Output{}, - Programs: []*core.Program{}, +func (wallet *Wallet) SendTransaction(tx *core.Transaction) error { + buf := new(bytes.Buffer) + if err := tx.Serialize(buf); err != nil { + return err } - return systemToken.Hash() + rawData := hex.EncodeToString(buf.Bytes()) + + _, err := jsonrpc.Call(jsonRpcUrl, util.Params{"data":rawData}) + return err } -func (wallet *wallet) removeLockedUTXOs(utxos []*sutil.UTXO) []*sutil.UTXO { +func (wallet *Wallet) removeLockedUTXOs(utxos []*sutil.UTXO) []*sutil.UTXO { var availableUTXOs []*sutil.UTXO var currentHeight = wallet.BestHeight() for _, utxo := range utxos { @@ -356,7 +343,7 @@ func InputFromUTXO(utxo *sutil.UTXO) *core.Input { return input } -func (wallet *wallet) newTransaction(redeemScript []byte, inputs []*core.Input, outputs []*core.Output) *core.Transaction { +func (wallet *Wallet) newTransaction(redeemScript []byte, inputs []*core.Input, outputs []*core.Output) *core.Transaction { // Create payload txPayload := &core.PayloadTransferAsset{} // Create attributes diff --git a/spvwallet/client/wallet/wallet.go b/wallet/client/wallet/wallet.go similarity index 98% rename from spvwallet/client/wallet/wallet.go rename to wallet/client/wallet/wallet.go index d553824..de40c16 100644 --- a/spvwallet/client/wallet/wallet.go +++ b/wallet/client/wallet/wallet.go @@ -3,7 +3,7 @@ package wallet import ( "fmt" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/client" + "github.com/elastos/Elastos.ELA.SPV/wallet/client" "github.com/urfave/cli" ) diff --git a/spvwallet/rpc/log.go b/wallet/log.go similarity index 97% rename from spvwallet/rpc/log.go rename to wallet/log.go index b7412c2..20ec0a1 100644 --- a/spvwallet/rpc/log.go +++ b/wallet/log.go @@ -1,4 +1,4 @@ -package rpc +package wallet import ( "github.com/elastos/Elastos.ELA.Utility/elalog" diff --git a/spvwallet/store/headers/cache.go b/wallet/store/headers/cache.go similarity index 100% rename from spvwallet/store/headers/cache.go rename to wallet/store/headers/cache.go diff --git a/spvwallet/store/headers/database.go b/wallet/store/headers/database.go similarity index 60% rename from spvwallet/store/headers/database.go rename to wallet/store/headers/database.go index 803abd9..80e9fd8 100644 --- a/spvwallet/store/headers/database.go +++ b/wallet/store/headers/database.go @@ -20,7 +20,8 @@ var _ database.Headers = (*Database)(nil) type Database struct { *sync.RWMutex *bolt.DB - cache *cache + cache *cache + newBlockHeader func() util.BlockHeader } var ( @@ -29,7 +30,7 @@ var ( KEYChainTip = []byte("ChainTip") ) -func New() (*Database, error) { +func NewDatabase(newBlockHeader func() util.BlockHeader) (*Database, error) { db, err := bolt.Open("headers.bin", 0644, &bolt.Options{InitialMmapSize: 5000000}) if err != nil { return nil, err @@ -48,9 +49,10 @@ func New() (*Database, error) { }) headers := &Database{ - RWMutex: new(sync.RWMutex), - DB: db, - cache: newHeaderCache(100), + RWMutex: new(sync.RWMutex), + DB: db, + cache: newHeaderCache(100), + newBlockHeader: newBlockHeader, } headers.initCache() @@ -58,34 +60,34 @@ func New() (*Database, error) { return headers, nil } -func (h *Database) initCache() { - best, err := h.GetBest() +func (d *Database) initCache() { + best, err := d.GetBest() if err != nil { return } - h.cache.tip = best + d.cache.tip = best headers := []*util.Header{best} for i := 0; i < 99; i++ { - sh, err := h.GetPrevious(best) + sh, err := d.GetPrevious(best) if err != nil { break } headers = append(headers, sh) } for i := len(headers) - 1; i >= 0; i-- { - h.cache.set(headers[i]) + d.cache.set(headers[i]) } } -func (h *Database) Put(header *util.Header, newTip bool) error { - h.Lock() - defer h.Unlock() +func (d *Database) Put(header *util.Header, newTip bool) error { + d.Lock() + defer d.Unlock() - h.cache.set(header) + d.cache.set(header) if newTip { - h.cache.tip = header + d.cache.tip = header } - return h.Update(func(tx *bolt.Tx) error { + return d.Update(func(tx *bolt.Tx) error { bytes, err := header.Serialize() if err != nil { @@ -108,25 +110,29 @@ func (h *Database) Put(header *util.Header, newTip bool) error { }) } -func (h *Database) GetPrevious(header *util.Header) (*util.Header, error) { +func (d *Database) GetPrevious(header *util.Header) (*util.Header, error) { + if header.Height == 0 { + return nil, fmt.Errorf("no more previous header") + } if header.Height == 1 { return &util.Header{TotalWork: new(big.Int)}, nil } - return h.Get(&header.Previous) + hash := header.Previous() + return d.Get(&hash) } -func (h *Database) Get(hash *common.Uint256) (header *util.Header, err error) { - h.RLock() - defer h.RUnlock() +func (d *Database) Get(hash *common.Uint256) (header *util.Header, err error) { + d.RLock() + defer d.RUnlock() - header, err = h.cache.get(hash) + header, err = d.cache.get(hash) if err == nil { return header, nil } - err = h.View(func(tx *bolt.Tx) error { + err = d.View(func(tx *bolt.Tx) error { - header, err = getHeader(tx, BKTHeaders, hash.Bytes()) + header, err = d.getHeader(tx, BKTHeaders, hash.Bytes()) if err != nil { return err } @@ -141,16 +147,16 @@ func (h *Database) Get(hash *common.Uint256) (header *util.Header, err error) { return header, err } -func (h *Database) GetBest() (header *util.Header, err error) { - h.RLock() - defer h.RUnlock() +func (d *Database) GetBest() (header *util.Header, err error) { + d.RLock() + defer d.RUnlock() - if h.cache.tip != nil { - return h.cache.tip, nil + if d.cache.tip != nil { + return d.cache.tip, nil } - err = h.View(func(tx *bolt.Tx) error { - header, err = getHeader(tx, BKTChainTip, KEYChainTip) + err = d.View(func(tx *bolt.Tx) error { + header, err = d.getHeader(tx, BKTChainTip, KEYChainTip) return err }) if err != nil { @@ -160,11 +166,11 @@ func (h *Database) GetBest() (header *util.Header, err error) { return header, err } -func (h *Database) Clear() error { - h.Lock() - defer h.Unlock() +func (d *Database) Clear() error { + d.Lock() + defer d.Unlock() - return h.Update(func(tx *bolt.Tx) error { + return d.Update(func(tx *bolt.Tx) error { err := tx.DeleteBucket(BKTHeaders) if err != nil { return err @@ -175,20 +181,21 @@ func (h *Database) Clear() error { } // Close db -func (h *Database) Close() error { - h.Lock() - err := h.DB.Close() +func (d *Database) Close() error { + d.Lock() + err := d.DB.Close() log.Debug("headers database closed") return err } -func getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*util.Header, error) { +func (d *Database) getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*util.Header, error) { headerBytes := tx.Bucket(bucket).Get(key) if headerBytes == nil { return nil, fmt.Errorf("Header %s does not exist in database", hex.EncodeToString(key)) } var header util.Header + header.BlockHeader = d.newBlockHeader() err := header.Deserialize(headerBytes) if err != nil { return nil, err diff --git a/spvwallet/store/headers/log.go b/wallet/store/headers/log.go similarity index 100% rename from spvwallet/store/headers/log.go rename to wallet/store/headers/log.go diff --git a/spvwallet/store/log.go b/wallet/store/log.go similarity index 72% rename from spvwallet/store/log.go rename to wallet/store/log.go index 68c9456..6b9b1e8 100644 --- a/spvwallet/store/log.go +++ b/wallet/store/log.go @@ -1,8 +1,8 @@ package store import ( - "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/headers" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/store/sqlite" + "github.com/elastos/Elastos.ELA.SPV/wallet/store/headers" + "github.com/elastos/Elastos.ELA.SPV/wallet/store/sqlite" "github.com/elastos/Elastos.ELA.Utility/elalog" ) diff --git a/spvwallet/store/sqlite/addrs.go b/wallet/store/sqlite/addrs.go similarity index 97% rename from spvwallet/store/sqlite/addrs.go rename to wallet/store/sqlite/addrs.go index aa53289..4c8ddcc 100644 --- a/spvwallet/store/sqlite/addrs.go +++ b/wallet/store/sqlite/addrs.go @@ -4,7 +4,7 @@ import ( "database/sql" "sync" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" ) diff --git a/spvwallet/store/sqlite/addrsbatch.go b/wallet/store/sqlite/addrsbatch.go similarity index 100% rename from spvwallet/store/sqlite/addrsbatch.go rename to wallet/store/sqlite/addrsbatch.go diff --git a/spvwallet/store/sqlite/database.go b/wallet/store/sqlite/database.go similarity index 98% rename from spvwallet/store/sqlite/database.go rename to wallet/store/sqlite/database.go index b7bc67a..1c59161 100644 --- a/spvwallet/store/sqlite/database.go +++ b/wallet/store/sqlite/database.go @@ -27,7 +27,7 @@ type database struct { stxos *stxos } -func New() (*database, error) { +func NewDatabase() (*database, error) { db, err := sql.Open(DriverName, DBName) if err != nil { fmt.Println("Open sqlite db error:", err) diff --git a/spvwallet/store/sqlite/databatch.go b/wallet/store/sqlite/databatch.go similarity index 99% rename from spvwallet/store/sqlite/databatch.go rename to wallet/store/sqlite/databatch.go index 6c44ec0..84d6f07 100644 --- a/spvwallet/store/sqlite/databatch.go +++ b/wallet/store/sqlite/databatch.go @@ -77,4 +77,4 @@ func (d *dataBatch) RollbackHeight(height uint32) error { // Rollback TXNs _, err = d.Exec("DELETE FROM TXNs WHERE Height=?", height) return err -} \ No newline at end of file +} diff --git a/spvwallet/store/sqlite/interface.go b/wallet/store/sqlite/interface.go similarity index 84% rename from spvwallet/store/sqlite/interface.go rename to wallet/store/sqlite/interface.go index 7552794..3e4167c 100644 --- a/spvwallet/store/sqlite/interface.go +++ b/wallet/store/sqlite/interface.go @@ -1,10 +1,10 @@ package sqlite import ( - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" ) type DataStore interface { @@ -69,16 +69,16 @@ type AddrsBatch interface { type Txs interface { // Put a new transaction to database - Put(txn *sutil.Tx) error + Put(tx *util.Tx) error // Fetch a raw tx and it's metadata given a hash - Get(txId *common.Uint256) (*sutil.Tx, error) + Get(txId *common.Uint256) (*util.Tx, error) // Fetch all transactions from database - GetAll() ([]*sutil.Tx, error) + GetAll() ([]*util.Tx, error) // Fetch all unconfirmed transactions. - GetAllUnconfirmed()([]*sutil.Tx, error) + GetAllUnconfirmed() ([]*util.Tx, error) // Delete a transaction from the db Del(txId *common.Uint256) error @@ -91,7 +91,7 @@ type TxsBatch interface { batch // Put a new transaction to database - Put(txn *sutil.Tx) error + Put(tx *util.Tx) error // Delete a transaction from the db Del(txId *common.Uint256) error @@ -102,7 +102,7 @@ type UTXOs interface { Put(utxo *sutil.UTXO) error // get a utxo from database - Get(op *core.OutPoint) (*sutil.UTXO, error) + Get(op *util.OutPoint) (*sutil.UTXO, error) // get utxos of the given address hash from database GetAddrAll(hash *common.Uint168) ([]*sutil.UTXO, error) @@ -111,7 +111,7 @@ type UTXOs interface { GetAll() ([]*sutil.UTXO, error) // delete a utxo from database - Del(outPoint *core.OutPoint) error + Del(outPoint *util.OutPoint) error // Batch return a UTXOsBatch. Batch() UTXOsBatch @@ -124,7 +124,7 @@ type UTXOsBatch interface { Put(utxo *sutil.UTXO) error // delete a utxo from database - Del(outPoint *core.OutPoint) error + Del(outPoint *util.OutPoint) error } type STXOs interface { @@ -132,7 +132,7 @@ type STXOs interface { Put(stxo *sutil.STXO) error // get a stxo from database - Get(op *core.OutPoint) (*sutil.STXO, error) + Get(op *util.OutPoint) (*sutil.STXO, error) // get stxos of the given address hash from database GetAddrAll(hash *common.Uint168) ([]*sutil.STXO, error) @@ -141,7 +141,7 @@ type STXOs interface { GetAll() ([]*sutil.STXO, error) // delete a stxo from database - Del(outPoint *core.OutPoint) error + Del(outPoint *util.OutPoint) error // Batch return a STXOsBatch. Batch() STXOsBatch @@ -154,5 +154,5 @@ type STXOsBatch interface { Put(stxo *sutil.STXO) error // delete a stxo from database - Del(outPoint *core.OutPoint) error + Del(outPoint *util.OutPoint) error } diff --git a/spvwallet/store/sqlite/log.go b/wallet/store/sqlite/log.go similarity index 100% rename from spvwallet/store/sqlite/log.go rename to wallet/store/sqlite/log.go diff --git a/spvwallet/store/sqlite/state.go b/wallet/store/sqlite/state.go similarity index 100% rename from spvwallet/store/sqlite/state.go rename to wallet/store/sqlite/state.go diff --git a/spvwallet/store/sqlite/stxos.go b/wallet/store/sqlite/stxos.go similarity index 87% rename from spvwallet/store/sqlite/stxos.go rename to wallet/store/sqlite/stxos.go index 3b6043f..c38783e 100644 --- a/spvwallet/store/sqlite/stxos.go +++ b/wallet/store/sqlite/stxos.go @@ -4,10 +4,10 @@ import ( "database/sql" "sync" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" ) const CreateSTXOsDB = `CREATE TABLE IF NOT EXISTS STXOs( @@ -53,12 +53,12 @@ func (s *stxos) Put(stxo *sutil.STXO) error { } // get a stxo from database -func (s *stxos) Get(outPoint *core.OutPoint) (*sutil.STXO, error) { +func (s *stxos) Get(op *util.OutPoint) (*sutil.STXO, error) { s.RLock() defer s.RUnlock() sql := `SELECT Value, LockTime, AtHeight, SpendHash, SpendHeight, Address FROM STXOs WHERE OutPoint=?` - row := s.QueryRow(sql, outPoint.Bytes()) + row := s.QueryRow(sql, op.Bytes()) var valueBytes []byte var lockTime uint32 var atHeight uint32 @@ -80,7 +80,7 @@ func (s *stxos) Get(outPoint *core.OutPoint) (*sutil.STXO, error) { return nil, err } - var utxo = sutil.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address} + var utxo = sutil.UTXO{Op: op, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address} spendHash, err := common.Uint256FromBytes(spendHashBytes) if err != nil { return nil, err @@ -133,7 +133,7 @@ func (s *stxos) getSTXOs(rows *sql.Rows) ([]*sutil.STXO, error) { return stxos, err } - outPoint, err := core.OutPointFromBytes(opBytes) + outPoint, err := util.OutPointFromBytes(opBytes) if err != nil { return stxos, err } @@ -145,7 +145,7 @@ func (s *stxos) getSTXOs(rows *sql.Rows) ([]*sutil.STXO, error) { if err != nil { return stxos, err } - var utxo = sutil.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address} + var utxo = sutil.UTXO{Op: outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address} spendHash, err := common.Uint256FromBytes(spendHashBytes) if err != nil { return stxos, err @@ -158,11 +158,11 @@ func (s *stxos) getSTXOs(rows *sql.Rows) ([]*sutil.STXO, error) { } // delete a stxo from database -func (s *stxos) Del(outPoint *core.OutPoint) error { +func (s *stxos) Del(op *util.OutPoint) error { s.Lock() defer s.Unlock() - _, err := s.Exec("DELETE FROM STXOs WHERE OutPoint=?", outPoint.Bytes()) + _, err := s.Exec("DELETE FROM STXOs WHERE OutPoint=?", op.Bytes()) return err } diff --git a/spvwallet/store/sqlite/stxosbatch.go b/wallet/store/sqlite/stxosbatch.go similarity index 84% rename from spvwallet/store/sqlite/stxosbatch.go rename to wallet/store/sqlite/stxosbatch.go index 45bc18b..ca6123b 100644 --- a/spvwallet/store/sqlite/stxosbatch.go +++ b/wallet/store/sqlite/stxosbatch.go @@ -4,9 +4,8 @@ import ( "database/sql" "sync" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" - - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" ) // Ensure stxos implement STXOs interface. @@ -34,7 +33,7 @@ func (s *stxosBatch) Put(stxo *sutil.STXO) error { } // delete a stxo from database -func (sb *stxosBatch) Del(outPoint *core.OutPoint) error { +func (sb *stxosBatch) Del(outPoint *util.OutPoint) error { sb.Lock() defer sb.Unlock() diff --git a/spvwallet/store/sqlite/txs.go b/wallet/store/sqlite/txs.go similarity index 68% rename from spvwallet/store/sqlite/txs.go rename to wallet/store/sqlite/txs.go index 24c0181..cef42a1 100644 --- a/spvwallet/store/sqlite/txs.go +++ b/wallet/store/sqlite/txs.go @@ -1,16 +1,14 @@ package sqlite import ( - "bytes" "database/sql" "math" "sync" "time" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" ) const CreateTxsDB = `CREATE TABLE IF NOT EXISTS Txs( @@ -37,23 +35,17 @@ func NewTxs(db *sql.DB, lock *sync.RWMutex) (*txs, error) { } // Put a new transaction to database -func (t *txs) Put(storeTx *sutil.Tx) error { +func (t *txs) Put(tx *util.Tx) error { t.Lock() defer t.Unlock() - buf := new(bytes.Buffer) - err := storeTx.Data.SerializeUnsigned(buf) - if err != nil { - return err - } - sql := `INSERT OR REPLACE INTO Txs(Hash, Height, Timestamp, RawData) VALUES(?,?,?,?)` - _, err = t.Exec(sql, storeTx.TxId.Bytes(), storeTx.Height, storeTx.Timestamp.Unix(), buf.Bytes()) + _, err := t.Exec(sql, tx.Hash.Bytes(), tx.Height, tx.Timestamp.Unix(), tx.RawData) return err } // Fetch a raw tx and it's metadata given a hash -func (t *txs) Get(txId *common.Uint256) (*sutil.Tx, error) { +func (t *txs) Get(txId *common.Uint256) (*util.Tx, error) { t.RLock() defer t.RUnlock() @@ -65,27 +57,23 @@ func (t *txs) Get(txId *common.Uint256) (*sutil.Tx, error) { if err != nil { return nil, err } - var tx core.Transaction - err = tx.DeserializeUnsigned(bytes.NewReader(rawData)) - if err != nil { - return nil, err - } - return &sutil.Tx{TxId: *txId, Height: height, Timestamp: time.Unix(timestamp, 0), Data: tx}, nil + return &util.Tx{Hash: *txId, Height: height, + Timestamp: time.Unix(timestamp, 0), RawData: rawData}, nil } // Fetch all transactions from database -func (t *txs) GetAll() ([]*sutil.Tx, error) { +func (t *txs) GetAll() ([]*util.Tx, error) { return t.GetAllFrom(math.MaxUint32) } // Fetch all unconfirmed transactions from database -func (t *txs) GetAllUnconfirmed() ([]*sutil.Tx, error) { +func (t *txs) GetAllUnconfirmed() ([]*util.Tx, error) { return t.GetAllFrom(0) } // Fetch all transactions from the given height -func (t *txs) GetAllFrom(height uint32) ([]*sutil.Tx, error) { +func (t *txs) GetAllFrom(height uint32) ([]*util.Tx, error) { t.RLock() defer t.RUnlock() @@ -93,7 +81,7 @@ func (t *txs) GetAllFrom(height uint32) ([]*sutil.Tx, error) { if height != math.MaxUint32 { sql += " WHERE Height=?" } - var txns []*sutil.Tx + var txns []*util.Tx rows, err := t.Query(sql, height) if err != nil { return txns, err @@ -115,13 +103,8 @@ func (t *txs) GetAllFrom(height uint32) ([]*sutil.Tx, error) { return txns, err } - var tx core.Transaction - err = tx.DeserializeUnsigned(bytes.NewReader(rawData)) - if err != nil { - return nil, err - } - - txns = append(txns, &sutil.Tx{TxId: *txId, Height: height, Timestamp: time.Unix(timestamp, 0), Data: tx}) + txns = append(txns, &util.Tx{Hash: *txId, Height: height, + Timestamp: time.Unix(timestamp, 0), RawData: rawData}) } return txns, nil diff --git a/spvwallet/store/sqlite/txsbatch.go b/wallet/store/sqlite/txsbatch.go similarity index 65% rename from spvwallet/store/sqlite/txsbatch.go rename to wallet/store/sqlite/txsbatch.go index 3a47510..7ea3a79 100644 --- a/spvwallet/store/sqlite/txsbatch.go +++ b/wallet/store/sqlite/txsbatch.go @@ -1,11 +1,10 @@ package sqlite import ( - "bytes" "database/sql" "sync" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" ) @@ -19,18 +18,12 @@ type txsBatch struct { } // Put a new transaction to database -func (t *txsBatch) Put(storeTx *sutil.Tx) error { +func (t *txsBatch) Put(tx *util.Tx) error { t.Lock() defer t.Unlock() - buf := new(bytes.Buffer) - err := storeTx.Data.SerializeUnsigned(buf) - if err != nil { - return err - } - sql := `INSERT OR REPLACE INTO Txs(Hash, Height, Timestamp, RawData) VALUES(?,?,?,?)` - _, err = t.Exec(sql, storeTx.TxId.Bytes(), storeTx.Height, storeTx.Timestamp.Unix(), buf.Bytes()) + _, err := t.Exec(sql, tx.Hash.Bytes(), tx.Height, tx.Timestamp.Unix(), tx.RawData) return err } diff --git a/spvwallet/store/sqlite/utxos.go b/wallet/store/sqlite/utxos.go similarity index 83% rename from spvwallet/store/sqlite/utxos.go rename to wallet/store/sqlite/utxos.go index 533bdd4..c4883d7 100644 --- a/spvwallet/store/sqlite/utxos.go +++ b/wallet/store/sqlite/utxos.go @@ -4,10 +4,10 @@ import ( "database/sql" "sync" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" ) const CreateUTXOsDB = `CREATE TABLE IF NOT EXISTS UTXOs( @@ -49,11 +49,11 @@ func (u *utxos) Put(utxo *sutil.UTXO) error { } // get a utxo from database -func (u *utxos) Get(outPoint *core.OutPoint) (*sutil.UTXO, error) { +func (u *utxos) Get(op *util.OutPoint) (*sutil.UTXO, error) { u.RLock() defer u.RUnlock() - row := u.QueryRow(`SELECT Value, LockTime, AtHeight, Address FROM UTXOs WHERE OutPoint=?`, outPoint.Bytes()) + row := u.QueryRow(`SELECT Value, LockTime, AtHeight, Address FROM UTXOs WHERE OutPoint=?`, op.Bytes()) var valueBytes []byte var lockTime uint32 var atHeight uint32 @@ -72,7 +72,7 @@ func (u *utxos) Get(outPoint *core.OutPoint) (*sutil.UTXO, error) { return nil, err } - return &sutil.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address}, nil + return &sutil.UTXO{Op: op, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address}, nil } // get utxos of the given script hash from database @@ -116,7 +116,7 @@ func (u *utxos) getUTXOs(rows *sql.Rows) ([]*sutil.UTXO, error) { return utxos, err } - outPoint, err := core.OutPointFromBytes(opBytes) + outPoint, err := util.OutPointFromBytes(opBytes) if err != nil { return utxos, err } @@ -128,7 +128,8 @@ func (u *utxos) getUTXOs(rows *sql.Rows) ([]*sutil.UTXO, error) { if err != nil { return utxos, err } - utxo := &sutil.UTXO{Op: *outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address} + utxo := &sutil.UTXO{Op: outPoint, Value: *value, LockTime: lockTime, + AtHeight: atHeight, Address: *address} utxos = append(utxos, utxo) } @@ -136,11 +137,11 @@ func (u *utxos) getUTXOs(rows *sql.Rows) ([]*sutil.UTXO, error) { } // delete a utxo from database -func (u *utxos) Del(outPoint *core.OutPoint) error { +func (u *utxos) Del(op *util.OutPoint) error { u.Lock() defer u.Unlock() - _, err := u.Exec("DELETE FROM UTXOs WHERE OutPoint=?", outPoint.Bytes()) + _, err := u.Exec("DELETE FROM UTXOs WHERE OutPoint=?", op.Bytes()) return err } diff --git a/spvwallet/store/sqlite/utxosbatch.go b/wallet/store/sqlite/utxosbatch.go similarity index 74% rename from spvwallet/store/sqlite/utxosbatch.go rename to wallet/store/sqlite/utxosbatch.go index fb6c242..59ba920 100644 --- a/spvwallet/store/sqlite/utxosbatch.go +++ b/wallet/store/sqlite/utxosbatch.go @@ -4,9 +4,8 @@ import ( "database/sql" "sync" - "github.com/elastos/Elastos.ELA.SPV/spvwallet/sutil" - - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" ) // Ensure utxos implement UTXOs interface. @@ -32,10 +31,10 @@ func (db *utxosBatch) Put(utxo *sutil.UTXO) error { } // delete a utxo from database -func (db *utxosBatch) Del(outPoint *core.OutPoint) error { +func (db *utxosBatch) Del(op *util.OutPoint) error { db.Lock() defer db.Unlock() - _, err := db.Exec("DELETE FROM UTXOs WHERE OutPoint=?", outPoint.Bytes()) + _, err := db.Exec("DELETE FROM UTXOs WHERE OutPoint=?", op.Bytes()) return err } diff --git a/spvwallet/sutil/addr.go b/wallet/sutil/addr.go similarity index 100% rename from spvwallet/sutil/addr.go rename to wallet/sutil/addr.go diff --git a/spvwallet/sutil/stxo.go b/wallet/sutil/stxo.go similarity index 100% rename from spvwallet/sutil/stxo.go rename to wallet/sutil/stxo.go diff --git a/spvwallet/sutil/utxo.go b/wallet/sutil/utxo.go similarity index 93% rename from spvwallet/sutil/utxo.go rename to wallet/sutil/utxo.go index b37ee33..71cfd1b 100644 --- a/spvwallet/sutil/utxo.go +++ b/wallet/sutil/utxo.go @@ -4,13 +4,14 @@ import ( "fmt" "sort" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" ) type UTXO struct { // Previous txid and output index - Op core.OutPoint + Op *util.OutPoint // The higher the better Value common.Fixed64 @@ -66,7 +67,7 @@ func (utxo *UTXO) IsEqual(alt *UTXO) bool { func NewUTXO(txId common.Uint256, height uint32, index int, value common.Fixed64, lockTime uint32, address common.Uint168) *UTXO { utxo := new(UTXO) - utxo.Op = *core.NewOutPoint(txId, uint16(index)) + utxo.Op = util.NewOutPoint(txId, uint16(index)) utxo.Value = value utxo.LockTime = lockTime utxo.AtHeight = height From ca82ff7895ab95aaf9e0c9fbee8c7a2e937d7691 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 23 Oct 2018 15:09:08 +0800 Subject: [PATCH 47/73] add ElaHeader and SideHeader types to compatible both MainChain and SideChain block header --- bloom/filter.go | 82 +++++++++-- bloom/{bloom.go => merkleblock.go} | 203 ++++++++++++++++++++------ bloom/merklebranch.go | 224 +++++++++++++++++++++++++++++ bloom/merklebranch_test.go | 101 +++++++++++++ client.go | 5 +- config.go | 29 +--- interface/blockheader.go | 24 +--- sdk/service.go | 14 +- spvwallet.go | 6 +- util/elaheader.go | 33 +++++ util/sideheader.go | 33 +++++ 11 files changed, 648 insertions(+), 106 deletions(-) rename bloom/{bloom.go => merkleblock.go} (51%) create mode 100644 bloom/merklebranch.go create mode 100644 bloom/merklebranch_test.go create mode 100644 util/elaheader.go create mode 100644 util/sideheader.go diff --git a/bloom/filter.go b/bloom/filter.go index 1358eb2..c16f0b8 100644 --- a/bloom/filter.go +++ b/bloom/filter.go @@ -4,10 +4,12 @@ import ( "math" "sync" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.SideChain/types" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/core" ) const ( @@ -169,7 +171,7 @@ func (bf *Filter) Matches(data []byte) bool { // outpoint and false if it definitely does not. // // This function MUST be called with the filter lock held. -func (bf *Filter) matchesOutPoint(outpoint *core.OutPoint) bool { +func (bf *Filter) matchesOutPoint(outpoint *util.OutPoint) bool { return bf.matches(outpoint.Bytes()) } @@ -177,7 +179,7 @@ func (bf *Filter) matchesOutPoint(outpoint *core.OutPoint) bool { // outpoint and false if it definitely does not. // // This function is safe for concurrent access. -func (bf *Filter) MatchesOutPoint(outpoint *core.OutPoint) bool { +func (bf *Filter) MatchesOutPoint(outpoint *util.OutPoint) bool { bf.mtx.Lock() match := bf.matchesOutPoint(outpoint) bf.mtx.Unlock() @@ -226,14 +228,14 @@ func (bf *Filter) AddHash(hash *common.Uint256) { // addOutPoint adds the passed tx outpoint to the bloom filter. // // This function MUST be called with the filter lock held. -func (bf *Filter) addOutPoint(outpoint *core.OutPoint) { +func (bf *Filter) addOutPoint(outpoint *util.OutPoint) { bf.add(outpoint.Bytes()) } // AddOutPoint adds the passed tx outpoint to the bloom filter. // // This function is safe for concurrent access. -func (bf *Filter) AddOutPoint(outpoint *core.OutPoint) { +func (bf *Filter) AddOutPoint(outpoint *util.OutPoint) { bf.mtx.Lock() bf.addOutPoint(outpoint) bf.mtx.Unlock() @@ -245,19 +247,72 @@ func (bf *Filter) AddOutPoint(outpoint *core.OutPoint) { // update flags set via the loaded filter if needed. // // This function MUST be called with the filter lock held. -func (bf *Filter) matchTxAndUpdate(txn *core.Transaction) bool { +func (bf *Filter) matchElaTxAndUpdate(tx *core.Transaction) bool { + // Check if the filter matches the hash of the tx. + // This is useful for finding transactions when they appear in a block. + hash := tx.Hash() + matched := bf.matches(hash[:]) + + for i, txOut := range tx.Outputs { + if !bf.matches(txOut.ProgramHash[:]) { + continue + } + + matched = true + bf.addOutPoint(util.NewOutPoint(tx.Hash(), uint16(i))) + } + + // Nothing more to do if a match has already been made. + if matched { + return true + } + + // At this point, the tx and none of the data elements in the + // public key scripts of its outputs matched. + + // Check if the filter matches any outpoints this tx spends + for _, txIn := range tx.Inputs { + op := txIn.Previous + if bf.matchesOutPoint(util.NewOutPoint(op.TxID, op.Index)) { + return true + } + } + + return false +} + +// MatchTxAndUpdate returns true if the bloom filter matches data within the +// passed tx, otherwise false is returned. If the filter does match +// the passed tx, it will also update the filter depending on the bloom +// update flags set via the loaded filter if needed. +// +// This function is safe for concurrent access. +func (bf *Filter) MatchElaTxAndUpdate(tx *core.Transaction) bool { + bf.mtx.Lock() + match := bf.matchElaTxAndUpdate(tx) + bf.mtx.Unlock() + return match +} + +// matchTxAndUpdate returns true if the bloom filter matches data within the +// passed tx, otherwise false is returned. If the filter does match +// the passed tx, it will also update the filter depending on the bloom +// update flags set via the loaded filter if needed. +// +// This function MUST be called with the filter lock held. +func (bf *Filter) matchSideTxAndUpdate(tx *types.Transaction) bool { // Check if the filter matches the hash of the tx. // This is useful for finding transactions when they appear in a block. - hash := txn.Hash() + hash := tx.Hash() matched := bf.matches(hash[:]) - for i, txOut := range txn.Outputs { + for i, txOut := range tx.Outputs { if !bf.matches(txOut.ProgramHash[:]) { continue } matched = true - bf.addOutPoint(core.NewOutPoint(txn.Hash(), uint16(i))) + bf.addOutPoint(util.NewOutPoint(tx.Hash(), uint16(i))) } // Nothing more to do if a match has already been made. @@ -269,8 +324,9 @@ func (bf *Filter) matchTxAndUpdate(txn *core.Transaction) bool { // public key scripts of its outputs matched. // Check if the filter matches any outpoints this tx spends - for _, txIn := range txn.Inputs { - if bf.matchesOutPoint(&txIn.Previous) { + for _, txIn := range tx.Inputs { + op := txIn.Previous + if bf.matchesOutPoint(util.NewOutPoint(op.TxID, op.Index)) { return true } } @@ -284,9 +340,9 @@ func (bf *Filter) matchTxAndUpdate(txn *core.Transaction) bool { // update flags set via the loaded filter if needed. // // This function is safe for concurrent access. -func (bf *Filter) MatchTxAndUpdate(tx *core.Transaction) bool { +func (bf *Filter) MatchSideTxAndUpdate(tx *types.Transaction) bool { bf.mtx.Lock() - match := bf.matchTxAndUpdate(tx) + match := bf.matchSideTxAndUpdate(tx) bf.mtx.Unlock() return match } diff --git a/bloom/bloom.go b/bloom/merkleblock.go similarity index 51% rename from bloom/bloom.go rename to bloom/merkleblock.go index 5d6f1ff..cde85b8 100644 --- a/bloom/bloom.go +++ b/bloom/merkleblock.go @@ -1,37 +1,158 @@ package bloom import ( + "errors" "fmt" - "github.com/elastos/Elastos.ELA.SPV/fprate" + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/elastos/Elastos.ELA.SideChain/types" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/core" ) -// Build a bloom filter by giving the interested addresses and outpoints -func BuildBloomFilter(addresses []*common.Uint168, outpoints []*util.OutPoint) *Filter { - elements := uint32(len(addresses) + len(outpoints)) +// mBlock is used to house intermediate information needed to generate a +// MerkleBlock according to a filter. +type mBlock struct { + NumTx uint32 + AllHashes []*common.Uint256 + FinalHashes []*common.Uint256 + MatchedBits []byte + Bits []byte +} + +// calcTreeWidth calculates and returns the the number of nodes (width) or a +// merkle tree at the given depth-first height. +func (m *mBlock) calcTreeWidth(height uint32) uint32 { + return (m.NumTx + (1 << height) - 1) >> height +} - filter := NewFilter(elements, 0, fprate.ReducedFalsePositiveRate) - for _, address := range addresses { - filter.Add(address.Bytes()) +// calcHash returns the hash for a sub-tree given a depth-first height and +// node position. +func (m *mBlock) calcHash(height, pos uint32) *common.Uint256 { + if height == 0 { + return m.AllHashes[pos] } - for _, op := range outpoints { - filter.Add(op.Bytes()) + var right *common.Uint256 + left := m.calcHash(height-1, pos*2) + if pos*2+1 < m.calcTreeWidth(height-1) { + right = m.calcHash(height-1, pos*2+1) + } else { + right = left } + return HashMerkleBranches(left, right) +} - return filter +// HashMerkleBranches takes two hashes, treated as the left and right tree +// nodes, and returns the hash of their concatenation. This is a helper +// function used to aid in the generation of a merkle tree. +func HashMerkleBranches(left *common.Uint256, right *common.Uint256) *common.Uint256 { + // Concatenate the left and right nodes. + var hash [common.UINT256SIZE * 2]byte + copy(hash[:common.UINT256SIZE], left[:]) + copy(hash[common.UINT256SIZE:], right[:]) + + newHash := common.Uint256(common.Sha256D(hash[:])) + return &newHash } -// Add a address into the given bloom filter -func FilterAddress(filter Filter, address *common.Uint168) { - filter.Add(address.Bytes()) +// traverseAndBuild builds a partial merkle tree using a recursive depth-first +// approach. As it calculates the hashes, it also saves whether or not each +// node is a parent node and a list of final hashes to be included in the +// merkle block. +func (m *mBlock) traverseAndBuild(height, pos uint32) { + // Determine whether this node is a parent of a matched node. + var isParent byte + for i := pos << height; i < (pos+1)< 1 { + height++ + } + + // Build the depth-first partial merkle tree. + mBlock.traverseAndBuild(height, 0) + + // Create and return the merkle block. + merkleBlock := &msg.MerkleBlock{ + Header: block.Header.BlockHeader, + Transactions: mBlock.NumTx, + Hashes: make([]*common.Uint256, 0, len(mBlock.FinalHashes)), + Flags: make([]byte, (len(mBlock.Bits)+7)/8), + } + for _, hash := range mBlock.FinalHashes { + merkleBlock.Hashes = append(merkleBlock.Hashes, hash) + } + for i := uint32(0); i < uint32(len(mBlock.Bits)); i++ { + merkleBlock.Flags[i/8] |= mBlock.Bits[i] << (i % 8) + } + + return merkleBlock, matchedIndexes } type merkleNode struct { @@ -71,29 +192,6 @@ func inDeadZone(pos, size uint32) bool { return pos > last } -func merkleParent(left *common.Uint256, right *common.Uint256) (*common.Uint256, error) { - // dupes can screw things up; CVE-2012-2459. check for them - if left != nil && right != nil && left.IsEqual(*right) { - return nil, fmt.Errorf("DUP HASH CRASH") - } - // if left child is nil, output nil. Need this for hard mode. - if left == nil { - return nil, fmt.Errorf("Left child is nil") - } - // if right is nil, hash left with itself - if right == nil { - right = left - } - - // Concatenate the left and right nodes - var sha [64]byte - copy(sha[:32], left[:]) - copy(sha[32:], right[:]) - - parent := common.Uint256(common.Sha256D(sha[:])) - return &parent, nil -} - // take in a merkle block, parse through it, and return txids indicated // If there's any problem return an error. Checks self-consistency only. // doing it with a stack instead of recursion. Because... @@ -130,7 +228,7 @@ func CheckMerkleBlock(m msg.MerkleBlock) ([]*common.Uint256, error) { // is current position in the tree's dead zone? partial parent if inDeadZone(pos, m.Transactions) { // create merkle parent from single side (left) - h, err := merkleParent(s[tip].h, nil) + h, err := MakeMerkleParent(s[tip].h, nil) if err != nil { return r, err } @@ -144,7 +242,7 @@ func CheckMerkleBlock(m msg.MerkleBlock) ([]*common.Uint256, error) { //fmt.Printf("nodes %d and %d combine into %d\n", // s[tip-1].p, s[tip].p, s[tip-2].p) // combine two filled nodes into parent node - h, err := merkleParent(s[tip-1].h, s[tip].h) + h, err := MakeMerkleParent(s[tip-1].h, s[tip].h) if err != nil { return r, err } @@ -204,3 +302,26 @@ func CheckMerkleBlock(m msg.MerkleBlock) ([]*common.Uint256, error) { } return nil, fmt.Errorf("ran out of things to do?") } + +func MakeMerkleParent(left *common.Uint256, right *common.Uint256) (*common.Uint256, error) { + // dupes can screw things up; CVE-2012-2459. check for them + if left != nil && right != nil && left.IsEqual(*right) { + return nil, errors.New("DUP HASH CRASH") + } + // if left child is nil, output nil. Need this for hard mode. + if left == nil { + return nil, errors.New("Left child is nil") + } + // if right is nil, hash left with itself + if right == nil { + right = left + } + + // Concatenate the left and right nodes + var sha [64]byte + copy(sha[:32], left[:]) + copy(sha[32:], right[:]) + + parent := common.Uint256(common.Sha256D(sha[:])) + return &parent, nil +} diff --git a/bloom/merklebranch.go b/bloom/merklebranch.go new file mode 100644 index 0000000..737687e --- /dev/null +++ b/bloom/merklebranch.go @@ -0,0 +1,224 @@ +package bloom + +import ( + "errors" + "fmt" + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" +) + +type MerkleBranch struct { + Branches []common.Uint256 + Index int +} + +func GetTxMerkleBranch(msg msg.MerkleBlock, txId *common.Uint256) (*MerkleBranch, error) { + mNodes := &merkleNodes{ + root: msg.Header.(util.BlockHeader).MerkleRoot(), + numTxs: msg.Transactions, + allNodes: make(map[uint32]merkleNode), + } + + mNodes.SetHashes(msg.Hashes) + mNodes.SetBits(msg.Flags) + + return mNodes.GetMerkleBranch(txId) +} + +type merkleNodes struct { + root common.Uint256 + numTxs uint32 + hashes []*common.Uint256 + bits []byte + txIndex uint32 + route []uint32 + allNodes map[uint32]merkleNode +} + +func (m *merkleNodes) GetMerkleBranch(txId *common.Uint256) (mb *MerkleBranch, err error) { + m.allNodes, err = m.getNodes() + if err != nil { + return nil, err + } + + m.calcTxIndex(txId) + m.calcBranchRoute() + + mb = new(MerkleBranch) + mb.Branches = make([]common.Uint256, 0, len(m.route)) + for i, index := range m.route { + mb.Branches = append(mb.Branches, *m.allNodes[index].h) + if index%2 == 0 { + mb.Index += 1 << uint32(i) + } + } + + return mb, nil +} + +func (m *merkleNodes) SetHashes(hashes []*common.Uint256) { + m.hashes = make([]*common.Uint256, 0, len(hashes)) + for _, hash := range hashes { + m.hashes = append(m.hashes, hash) + } +} + +func (m *merkleNodes) SetBits(flags []byte) { + m.bits = make([]byte, uint32(len(flags))*8) + for i := uint32(0); i < uint32(len(flags))*8; i++ { + m.bits[i] = (flags[i/8] >> (i % 8)) & 1 + } +} + +func (m merkleNodes) getNodes() (map[uint32]merkleNode, error) { + if m.numTxs == 0 { + return nil, fmt.Errorf("No transactions in merkleblock") + } + if len(m.bits) == 0 { + return nil, fmt.Errorf("No flag bits") + } + var s []merkleNode // the stack + var r = make(map[uint32]merkleNode) // the return nodes + // set initial position to root of merkle tree + msb := nextPowerOfTwo(m.numTxs) // most significant bit possible + pos := (msb << 1) - 2 // current position in tree + + var tip int + // main loop + for { + tip = len(s) - 1 // slice position of stack tip + // First check if stack operations can be performed + // is stack one filled item? that's complete. + if tip == 0 && s[0].h != nil { + if s[0].h.IsEqual(m.root) { + return r, nil + } + return nil, fmt.Errorf("computed root %s but expect %s\n", + s[0].h.String(), m.root.String()) + } + // is current position in the tree's dead zone? partial parent + if inDeadZone(pos, m.numTxs) { + // create merkle parent from single side (left) + h, err := MakeMerkleParent(s[tip].h, nil) + if err != nil { + return r, err + } + s[tip-1].h = h + s = s[:tip] // remove 1 from stack + pos = s[tip-1].p | 1 // move position to parent's sibling + r[s[tip-1].p] = s[tip-1] + continue + } + // does stack have 3+ items? and are last 2 items filled? + if tip > 1 && s[tip-1].h != nil && s[tip].h != nil { + // combine two filled nodes into parent node + h, err := MakeMerkleParent(s[tip-1].h, s[tip].h) + if err != nil { + return r, err + } + s[tip-2].h = h + // remove children + s = s[:tip-1] + // move position to parent's sibling + pos = s[tip-2].p | 1 + r[s[tip-2].p] = s[tip-2] + continue + } + + // no stack ops to perform, so make new node from message hashes + if len(m.hashes) == 0 { + return nil, fmt.Errorf("Ran out of hashes at position %d.", pos) + } + if len(m.bits) == 0 { + return nil, fmt.Errorf("Ran out of bits.") + } + var n merkleNode // make new node + n.p = pos // set current position for new node + + if pos&msb != 0 { // upper non-txid hash + if m.bits[0] == 0 { // flag bit says fill node + n.h = m.hashes[0] // copy hash from message + m.hashes = m.hashes[1:] // pop off message + if pos&1 != 0 { // right side; ascend + pos = pos>>1 | msb + } else { // left side, go to sibling + pos |= 1 + } + r[n.p] = n + } else { // flag bit says skip; put empty on stack and descend + pos = (pos ^ msb) << 1 // descend to left + } + s = append(s, n) // push new node on stack + } else { // bottom row txid; flag bit indicates tx of interest + if pos >= m.numTxs { + // this can't happen because we check deadzone above... + return nil, fmt.Errorf("got into an invalid txid node") + } + n.h = m.hashes[0] // copy hash from message + m.hashes = m.hashes[1:] // pop off message + if pos&1 == 0 { // left side, go to sibling + pos |= 1 + } // if on right side we don't move; stack ops will move next + r[n.p] = n + s = append(s, n) // push new node onto the stack + } + + // done with pushing onto stack; advance flag bit + m.bits = m.bits[1:] + } + return nil, fmt.Errorf("ran out of things to do?") +} + +func (m *merkleNodes) calcTxIndex(txId *common.Uint256) error { + width := m.calcTreeWidth(0) + for _, node := range m.allNodes { + if node.p > width { + continue + } + if *node.h == *txId { + m.txIndex = node.p + return nil + } + } + return errors.New("tx index not found") +} + +func (m *merkleNodes) calcBranchRoute() { + depth := treeDepth(m.numTxs) + for height := uint32(0); height < depth; height++ { + if m.inDeadZone(height) { + m.route = append(m.route, m.calcNodeIndex(height, m.txIndex>>height)) + continue + } + if m.isLeftLeaf(height) { + m.route = append(m.route, m.calcNodeIndex(height, (m.txIndex>>height)+1)) + } else { + m.route = append(m.route, m.calcNodeIndex(height, (m.txIndex>>height)-1)) + } + } +} + +func (m *merkleNodes) inDeadZone(height uint32) bool { + index := m.txIndex >> height + return index == m.calcTreeWidth(height)-1 && index%2 == 0 +} + +func (m *merkleNodes) isLeftLeaf(height uint32) bool { + index := m.txIndex >> height + return index%2 == 0 +} + +func (m *merkleNodes) calcTreeWidth(height uint32) uint32 { + return (m.numTxs + (1 << height) - 1) >> height +} + +func (m *merkleNodes) calcNodeIndex(height, pos uint32) uint32 { + msb := nextPowerOfTwo(m.numTxs)<<1 - 2 + height = treeDepth(m.numTxs) - height + for i := uint32(1); i <= height; i++ { + msb -= 1 << i + } + return msb + pos +} diff --git a/bloom/merklebranch_test.go b/bloom/merklebranch_test.go new file mode 100644 index 0000000..07175f6 --- /dev/null +++ b/bloom/merklebranch_test.go @@ -0,0 +1,101 @@ +package bloom + +import ( + "crypto/rand" + "fmt" + "os" + "testing" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/auxpow" + "github.com/elastos/Elastos.ELA/core" +) + +func TestMerkleBlock_GetTxMerkleBranch(t *testing.T) { + for txs := uint32(1); txs < 1<<10; txs++ { + run(txs) + fmt.Println("GetTxMerkleBranch() with txs:", txs, "PASSED") + } +} + +func run(txs uint32) { + mBlock := mBlock{ + NumTx: txs, + AllHashes: make([]*common.Uint256, 0, txs), + MatchedBits: make([]byte, 0, txs), + } + + matches := randMatches(txs) + for i := uint32(0); i < txs; i++ { + if matches[i] { + mBlock.MatchedBits = append(mBlock.MatchedBits, 0x01) + } else { + mBlock.MatchedBits = append(mBlock.MatchedBits, 0x00) + } + mBlock.AllHashes = append(mBlock.AllHashes, randHash()) + } + + // Calculate the number of merkle branches (height) in the tree. + height := uint32(0) + for mBlock.calcTreeWidth(height) > 1 { + height++ + } + + // Build the depth-first partial merkle tree. + mBlock.traverseAndBuild(height, 0) + + merkleRoot := *mBlock.calcHash(treeDepth(txs), 0) + // Create and return the merkle block. + merkleBlock := msg.MerkleBlock{ + Header: &core.Header{ + MerkleRoot: merkleRoot, + }, + Transactions: mBlock.NumTx, + Hashes: make([]*common.Uint256, 0, len(mBlock.FinalHashes)), + Flags: make([]byte, (len(mBlock.Bits)+7)/8), + } + for _, hash := range mBlock.FinalHashes { + merkleBlock.Hashes = append(merkleBlock.Hashes, hash) + } + for i := uint32(0); i < uint32(len(mBlock.Bits)); i++ { + merkleBlock.Flags[i/8] |= mBlock.Bits[i] << (i % 8) + } + + txIds, err := CheckMerkleBlock(merkleBlock) + if err != nil { + fmt.Println(err.Error()) + os.Exit(0) + } + + for i := range txIds { + mb, err := GetTxMerkleBranch(merkleBlock, txIds[i]) + if err != nil { + fmt.Println(err.Error()) + os.Exit(0) + } + + calcRoot := auxpow.GetMerkleRoot(*txIds[i], mb.Branches, mb.Index) + if merkleRoot == calcRoot { + } else { + fmt.Println("Merkle root not match, expect %s result %s", merkleRoot.String(), calcRoot.String()) + os.Exit(0) + } + } +} + +func randHash() *common.Uint256 { + var hash common.Uint256 + rand.Read(hash[:]) + return &hash +} + +func randMatches(hashes uint32) map[uint32]bool { + matches := make(map[uint32]bool) + b := make([]byte, 1) + for i := uint32(0); i < hashes; i++ { + rand.Read(b) + matches[i] = b[0]%2 == 1 + } + return matches +} diff --git a/client.go b/client.go index 5ce265d..6acd59f 100644 --- a/client.go +++ b/client.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.SPV/wallet" "github.com/elastos/Elastos.ELA.Utility/common" @@ -13,7 +14,9 @@ var Version string func main() { url := fmt.Sprint("http://127.0.0.1:", config.JsonRpcPort, "/spvwallet/") - wallet.RunClient(Version, url, getSystemAssetId(), newBlockHeader) + wallet.RunClient(Version, url, getSystemAssetId(), func() util.BlockHeader { + return util.NewElaHeader(&core.Header{}) + }) } func getSystemAssetId() common.Uint256 { diff --git a/config.go b/config.go index d06aaff..9cfbc03 100644 --- a/config.go +++ b/config.go @@ -4,11 +4,10 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" "io/ioutil" "os" + + "github.com/elastos/Elastos.ELA.Utility/common" ) const ( @@ -58,27 +57,3 @@ func loadConfig() *Config { return &c } - -type blockHeader struct { - *core.Header -} - -func (h *blockHeader) Previous() common.Uint256 { - return h.Header.Previous -} - -func (h *blockHeader) Bits() uint32 { - return h.Header.Bits -} - -func (h *blockHeader) MerkleRoot() common.Uint256 { - return h.Header.MerkleRoot -} - -func (h *blockHeader) PowHash() common.Uint256 { - return h.AuxPow.ParBlockHeader.Hash() -} - -func newBlockHeader() util.BlockHeader { - return &blockHeader{Header: &core.Header{}} -} diff --git a/interface/blockheader.go b/interface/blockheader.go index ff470d0..5f72640 100644 --- a/interface/blockheader.go +++ b/interface/blockheader.go @@ -10,28 +10,8 @@ import ( "github.com/elastos/Elastos.ELA/core" ) -type blockHeader struct { - *core.Header -} - -func (h *blockHeader) Previous() common.Uint256 { - return h.Header.Previous -} - -func (h *blockHeader) Bits() uint32 { - return h.Header.Bits -} - -func (h *blockHeader) MerkleRoot() common.Uint256 { - return h.Header.MerkleRoot -} - -func (h *blockHeader) PowHash() common.Uint256 { - return h.AuxPow.ParBlockHeader.Hash() -} - func newBlockHeader() util.BlockHeader { - return &blockHeader{Header: &core.Header{}} + return util.NewElaHeader(&core.Header{}) } // GenesisHeader creates a specific genesis header by the given @@ -107,5 +87,5 @@ func GenesisHeader(foundation *common.Uint168) util.BlockHeader { } header.MerkleRoot, _ = crypto.ComputeRoot(hashes) - return &blockHeader{Header: &header} + return util.NewElaHeader(&header) } diff --git a/sdk/service.go b/sdk/service.go index cd1392e..de0f97f 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -131,7 +131,19 @@ func (s *service) start() { } func (s *service) updateFilter() *bloom.Filter { - return bloom.BuildBloomFilter(s.cfg.GetFilterData()) + addresses, outpoints := s.cfg.GetFilterData() + elements := uint32(len(addresses) + len(outpoints)) + + filter := bloom.NewFilter(elements, 0, 0) + for _, address := range addresses { + filter.Add(address.Bytes()) + } + + for _, op := range outpoints { + filter.Add(op.Bytes()) + } + + return filter } func (s *service) makeEmptyMessage(cmd string) (p2p.Message, error) { diff --git a/spvwallet.go b/spvwallet.go index bfcf5ae..d8618fc 100644 --- a/spvwallet.go +++ b/spvwallet.go @@ -318,6 +318,10 @@ func NewWallet() (*spvwallet, error) { return &w, nil } +func newBlockHeader() util.BlockHeader { + return util.NewElaHeader(&core.Header{}) +} + func newTransaction() util.Transaction { return new(core.Transaction) } @@ -395,5 +399,5 @@ func GenesisHeader() util.BlockHeader { } header.MerkleRoot, _ = crypto.ComputeRoot(hashes) - return &blockHeader{Header: &header} + return util.NewElaHeader(&header) } diff --git a/util/elaheader.go b/util/elaheader.go new file mode 100644 index 0000000..90c72c1 --- /dev/null +++ b/util/elaheader.go @@ -0,0 +1,33 @@ +package util + +import ( + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" +) + +// Ensure SideHeader implement BlockHeader interface. +var _ BlockHeader = (*ElaHeader)(nil) + +type ElaHeader struct { + *core.Header +} + +func (h *ElaHeader) Previous() common.Uint256 { + return h.Header.Previous +} + +func (h *ElaHeader) Bits() uint32 { + return h.Header.Bits +} + +func (h *ElaHeader) MerkleRoot() common.Uint256 { + return h.Header.MerkleRoot +} + +func (h *ElaHeader) PowHash() common.Uint256 { + return h.AuxPow.ParBlockHeader.Hash() +} + +func NewElaHeader(orgHeader *core.Header) BlockHeader { + return &ElaHeader{Header: orgHeader} +} diff --git a/util/sideheader.go b/util/sideheader.go new file mode 100644 index 0000000..cbf28a8 --- /dev/null +++ b/util/sideheader.go @@ -0,0 +1,33 @@ +package util + +import ( + "github.com/elastos/Elastos.ELA.SideChain/types" + "github.com/elastos/Elastos.ELA.Utility/common" +) + +// Ensure SideHeader implement BlockHeader interface. +var _ BlockHeader = (*SideHeader)(nil) + +type SideHeader struct { + *types.Header +} + +func (h *SideHeader) Previous() common.Uint256 { + return h.Header.Previous +} + +func (h *SideHeader) Bits() uint32 { + return h.Header.Bits +} + +func (h *SideHeader) MerkleRoot() common.Uint256 { + return h.Header.MerkleRoot +} + +func (h *SideHeader) PowHash() common.Uint256 { + return h.SideAuxPow.MainBlockHeader.Hash() +} + +func NewSideHeader(orgHeader *types.Header) BlockHeader { + return &SideHeader{Header: orgHeader} +} From decd54442ac384639b24ebb612ddeb9a85c9fd00 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Wed, 24 Oct 2018 17:16:10 +0800 Subject: [PATCH 48/73] fix query UTXO database failure issue --- wallet/store/sqlite/stxos.go | 16 ++++++++++------ wallet/store/sqlite/utxos.go | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/wallet/store/sqlite/stxos.go b/wallet/store/sqlite/stxos.go index c38783e..b0a549c 100644 --- a/wallet/store/sqlite/stxos.go +++ b/wallet/store/sqlite/stxos.go @@ -45,8 +45,8 @@ func (s *stxos) Put(stxo *sutil.STXO) error { if err != nil { return err } - sql := `INSERT OR REPLACE INTO STXOs(OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight, Address) - VALUES(?,?,?,?,?,?,?)` + sql := `INSERT OR REPLACE INTO STXOs(OutPoint, Value, LockTime, AtHeight, + SpendHash, SpendHeight, Address) VALUES(?,?,?,?,?,?,?)` _, err = s.Exec(sql, stxo.Op.Bytes(), valueBytes, stxo.LockTime, stxo.AtHeight, stxo.SpendTxId.Bytes(), stxo.SpendHeight, stxo.Address.Bytes()) return err @@ -94,7 +94,8 @@ func (s *stxos) GetAddrAll(hash *common.Uint168) ([]*sutil.STXO, error) { s.RLock() defer s.RUnlock() - sql := "SELECT OutPoint, Value, LockTime, AtHeight, SpendHash, SpendHeight, Address FROM STXOs WHERE Address=?" + sql := `SELECT OutPoint, Value, LockTime, AtHeight, SpendHash, + SpendHeight, Address FROM STXOs WHERE Address=?` rows, err := s.Query(sql, hash.Bytes()) if err != nil { return nil, err @@ -128,7 +129,8 @@ func (s *stxos) getSTXOs(rows *sql.Rows) ([]*sutil.STXO, error) { var spendHashBytes []byte var spendHeight uint32 var addressBytes []byte - err := rows.Scan(&opBytes, &valueBytes, &lockTime, &atHeight, &spendHashBytes, &spendHeight, &addressBytes) + err := rows.Scan(&opBytes, &valueBytes, &lockTime, &atHeight, + &spendHashBytes, &spendHeight, &addressBytes) if err != nil { return stxos, err } @@ -145,13 +147,15 @@ func (s *stxos) getSTXOs(rows *sql.Rows) ([]*sutil.STXO, error) { if err != nil { return stxos, err } - var utxo = sutil.UTXO{Op: outPoint, Value: *value, LockTime: lockTime, AtHeight: atHeight, Address: *address} + var utxo = sutil.UTXO{Op: outPoint, Value: *value, LockTime: lockTime, + AtHeight: atHeight, Address: *address} spendHash, err := common.Uint256FromBytes(spendHashBytes) if err != nil { return stxos, err } - stxos = append(stxos, &sutil.STXO{UTXO: utxo, SpendTxId: *spendHash, SpendHeight: spendHeight}) + stxos = append(stxos, &sutil.STXO{UTXO: utxo, SpendTxId: *spendHash, + SpendHeight: spendHeight}) } return stxos, nil diff --git a/wallet/store/sqlite/utxos.go b/wallet/store/sqlite/utxos.go index c4883d7..e3dedc3 100644 --- a/wallet/store/sqlite/utxos.go +++ b/wallet/store/sqlite/utxos.go @@ -81,7 +81,7 @@ func (u *utxos) GetAddrAll(hash *common.Uint168) ([]*sutil.UTXO, error) { defer u.RUnlock() rows, err := u.Query( - "SELECT OutPoint, Value, LockTime, AtHeight, Address FROM UTXOs WHERE ScriptHash=?", hash.Bytes()) + "SELECT OutPoint, Value, LockTime, AtHeight, Address FROM UTXOs WHERE Address=?", hash.Bytes()) if err != nil { return nil, err } From 97fd35c371d6fb94f82e7ff03fae060eca6d75ed Mon Sep 17 00:00:00 2001 From: AlexPan Date: Wed, 24 Oct 2018 17:16:51 +0800 Subject: [PATCH 49/73] fix client JSON-RPC not working issue --- Makefile | 14 ++++++++++---- client.go | 2 +- log.go | 2 +- spvwallet.go | 15 +++++++++++++-- wallet/client/common.go | 2 +- wallet/client/wallet.go | 13 ++++++------- 6 files changed, 32 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 81e6728..4f5988d 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,15 @@ BUILD=go build VERSION := $(shell git describe --abbrev=4 --dirty --always --tags) -BUILD_SPV_CLI =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o ela-wallet log.go config.go client.go -BUILD_SPV_SERVICE =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o service log.go config.go spvwallet.go main.go +BUILD_CLIENT =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o ela-wallet log.go config.go client.go +BUILD_SERVICE =$(BUILD) -ldflags "-X main.Version=$(VERSION)" -o service log.go config.go spvwallet.go main.go all: - $(BUILD_SPV_CLI) - $(BUILD_SPV_SERVICE) \ No newline at end of file + $(BUILD_CLIENT) + $(BUILD_SERVICE) + +client: + $(BUILD_CLIENT) + +service: + $(BUILD_SERVICE) \ No newline at end of file diff --git a/client.go b/client.go index 6acd59f..d077bec 100644 --- a/client.go +++ b/client.go @@ -13,7 +13,7 @@ import ( var Version string func main() { - url := fmt.Sprint("http://127.0.0.1:", config.JsonRpcPort, "/spvwallet/") + url := fmt.Sprint("http://127.0.0.1:", config.JsonRpcPort, "/spvwallet") wallet.RunClient(Version, url, getSystemAssetId(), func() util.BlockHeader { return util.NewElaHeader(&core.Header{}) }) diff --git a/log.go b/log.go index f2177b9..8fbf1dd 100644 --- a/log.go +++ b/log.go @@ -38,7 +38,7 @@ var ( bcdblog = backend.Logger("BCDB", level) synclog = backend.Logger("SYNC", level) peerlog = backend.Logger("PEER", level) - spvslog = backend.Logger("SPVS", level) + spvslog = backend.Logger("SPVS", elalog.LevelInfo) srvrlog = backend.Logger("SRVR", level) rpcslog = backend.Logger("RPCS", level) waltlog = backend.Logger("WALT", level) diff --git a/spvwallet.go b/spvwallet.go index d8618fc..a75989a 100644 --- a/spvwallet.go +++ b/spvwallet.go @@ -15,6 +15,7 @@ import ( "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" + "github.com/elastos/Elastos.ELA.Utility/http/jsonrpc" httputil "github.com/elastos/Elastos.ELA.Utility/http/util" "github.com/elastos/Elastos.ELA/core" ) @@ -240,16 +241,18 @@ func (b *txBatch) Commit() error { // Functions for RPC service. func (w *spvwallet) notifyNewAddress(params httputil.Params) (interface{}, error) { - data, ok := params.String("addr") + addrStr, ok := params.String("addr") if !ok { return nil, ErrInvalidParameter } - _, err := hex.DecodeString(data) + address, err := common.Uint168FromAddress(addrStr) if err != nil { return nil, err } + waltlog.Debugf("receive notifyNewAddress %s", address) + // Reload address filter to include new address w.loadAddrFilter() @@ -315,6 +318,14 @@ func NewWallet() (*spvwallet, error) { return nil, err } + s := jsonrpc.NewServer(&jsonrpc.Config{ + Path: "/spvwallet", + ServePort: config.JsonRpcPort, + }) + s.RegisterAction("notifynewaddress", w.notifyNewAddress, "addr") + s.RegisterAction("sendrawtransaction", w.sendTransaction, "data") + go s.Start() + return &w, nil } diff --git a/wallet/client/common.go b/wallet/client/common.go index 2baa163..9524c3a 100644 --- a/wallet/client/common.go +++ b/wallet/client/common.go @@ -125,7 +125,7 @@ func ShowAccounts(addrs []*sutil.Addr, newAddr *common.Uint168, wallet *Wallet) locked := common.Fixed64(0) UTXOs, err := wallet.GetAddressUTXOs(addr.Hash()) if err != nil { - return errors.New("get " + addr.String() + " UTXOs failed") + return fmt.Errorf("get %s UTXOs failed, %s", addr, err) } for _, utxo := range UTXOs { if utxo.LockTime >= currentHeight || utxo.AtHeight == 0 { diff --git a/wallet/client/wallet.go b/wallet/client/wallet.go index 01b73e7..94f61af 100644 --- a/wallet/client/wallet.go +++ b/wallet/client/wallet.go @@ -4,26 +4,25 @@ import ( "bytes" "encoding/hex" "errors" - util2 "github.com/elastos/Elastos.ELA.SPV/util" "math" "math/rand" "strconv" "github.com/elastos/Elastos.ELA.SPV/sdk" + "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.SPV/wallet/client/database" "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/crypto" "github.com/elastos/Elastos.ELA.Utility/http/jsonrpc" - "github.com/elastos/Elastos.ELA.Utility/http/util" "github.com/elastos/Elastos.ELA/core" ) var ( jsonRpcUrl string sysAssetId common.Uint256 - newHeader func() util2.BlockHeader + newHeader func() util.BlockHeader ) type Transfer struct { @@ -36,7 +35,7 @@ type Wallet struct { *Keystore } -func Setup(rpcUrl string, assetId common.Uint256, newBlockHeader func() util2.BlockHeader) { +func Setup(rpcUrl string, assetId common.Uint256, newBlockHeader func() util.BlockHeader) { jsonRpcUrl = rpcUrl sysAssetId = assetId newHeader = newBlockHeader @@ -91,7 +90,7 @@ func (wallet *Wallet) NewSubAccount(password []byte) (*common.Uint168, error) { } // Notify SPV service to reload bloom filter with the new address - jsonrpc.Call(jsonRpcUrl, util.Params{"addr": account.ProgramHash().String()}) + jsonrpc.CallArray(jsonRpcUrl, "notifynewaddress", account.ProgramHash().String()) return account.ProgramHash(), nil } @@ -113,7 +112,7 @@ func (wallet *Wallet) AddMultiSignAccount(M uint, publicKeys ...*crypto.PublicKe } // Notify SPV service to reload bloom filter with the new address - jsonrpc.Call(jsonRpcUrl, util.Params{"addr": programHash.String()}) + jsonrpc.CallArray(jsonRpcUrl, "notifynewaddress", programHash.String()) return programHash, nil } @@ -313,7 +312,7 @@ func (wallet *Wallet) SendTransaction(tx *core.Transaction) error { } rawData := hex.EncodeToString(buf.Bytes()) - _, err := jsonrpc.Call(jsonRpcUrl, util.Params{"data":rawData}) + _, err := jsonrpc.CallArray(jsonRpcUrl, "sendrawtransaction", rawData) return err } From 85b0f927563eba70cd5cd73e8f1386e4d00be20b Mon Sep 17 00:00:00 2001 From: AlexPan Date: Fri, 26 Oct 2018 15:31:29 +0800 Subject: [PATCH 50/73] add GetTransaction() and GetTransactionIds() method for SPVService interface --- interface/interface.go | 6 ++++++ interface/spvservice.go | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/interface/interface.go b/interface/interface.go index e59c3c3..933a444 100644 --- a/interface/interface.go +++ b/interface/interface.go @@ -60,6 +60,12 @@ type SPVService interface { // Send a transaction to the P2P network SendTransaction(core.Transaction) error + // GetTransaction query a transaction by it's hash. + GetTransaction(txId *common.Uint256) (*core.Transaction, error) + + // GetTransactionIds query all transaction hashes on the given block height. + GetTransactionIds(height uint32) ([]*common.Uint256, error) + // Get headers database HeaderStore() database.Headers diff --git a/interface/spvservice.go b/interface/spvservice.go index 891325a..dfca0c6 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -137,6 +137,25 @@ func (s *spvservice) SendTransaction(tx core.Transaction) error { return s.IService.SendTransaction(&tx) } +func (s *spvservice) GetTransaction(txId *common.Uint256) (*core.Transaction, error) { + utx, err := s.db.Txs().Get(txId) + if err != nil { + return nil, err + } + + var tx core.Transaction + err = tx.Deserialize(bytes.NewReader(utx.RawData)) + if err != nil { + return nil, err + } + + return &tx, nil +} + +func (s *spvservice) GetTransactionIds(height uint32) ([]*common.Uint256, error) { + return s.db.Txs().GetIds(height) +} + func (s *spvservice) HeaderStore() database.Headers { return s.headers } From e86b93cdc10d121ca3bdc7d639e4112afb7ac7a8 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 27 Oct 2018 10:35:00 +0800 Subject: [PATCH 51/73] fix interface/store/txs.go can not save and read txIds issue --- interface/interface_test.go | 65 +++++++++++++++++++++--- interface/spvservice.go | 6 +-- interface/store/databatch.go | 2 +- interface/store/txs.go | 98 +++++++++++++++++++++++------------- interface/store/txs_test.go | 46 +++++++++++++++++ interface/store/txsbatch.go | 6 +-- 6 files changed, 173 insertions(+), 50 deletions(-) create mode 100644 interface/store/txs_test.go diff --git a/interface/interface_test.go b/interface/interface_test.go index 5c65dc9..cb03ce3 100644 --- a/interface/interface_test.go +++ b/interface/interface_test.go @@ -2,6 +2,7 @@ package _interface import ( "fmt" + "math/rand" "os" "testing" "time" @@ -20,11 +21,12 @@ import ( "github.com/elastos/Elastos.ELA.Utility/p2p/connmgr" "github.com/elastos/Elastos.ELA.Utility/p2p/server" "github.com/elastos/Elastos.ELA/core" + "github.com/stretchr/testify/assert" ) -var service SPVService - type TxListener struct { + t *testing.T + service SPVService address string txType core.TransactionType flags uint64 @@ -44,12 +46,32 @@ func (l *TxListener) Flags() uint64 { func (l *TxListener) Notify(id common.Uint256, proof bloom.MerkleProof, tx core.Transaction) { fmt.Printf("Receive notify ID: %s, Type: %s\n", id.String(), tx.TxType.Name()) - err := service.VerifyTransaction(proof, tx) - if err != nil { - fmt.Println("Verify transaction error:", err) + err := l.service.VerifyTransaction(proof, tx) + if !assert.NoError(l.t, err) { + l.t.FailNow() + } + + + txIds, err := l.service.GetTransactionIds(proof.Height) + if !assert.NotNil(l.t, tx) { + l.t.FailNow() + } + if !assert.NoError(l.t, err) { + l.t.FailNow() } + + for _, txId := range txIds { + tx, err := l.service.GetTransaction(txId) + if !assert.NotNil(l.t, tx) { + l.t.FailNow() + } + if !assert.NoError(l.t, err) { + l.t.FailNow() + } + } + // Submit transaction receipt - service.SubmitTransactionReceipt(id, tx.Hash()) + l.service.SubmitTransactionReceipt(id, tx.Hash()) } func (l *TxListener) Rollback(height uint32) {} @@ -148,13 +170,17 @@ func TestNewSPVService(t *testing.T) { } confirmedListener := &TxListener{ - address: "ENTogr92671PKrMmtWo3RLiYXfBTXUe13Z", + t: t, + service: service, + address: "8ZNizBf4KhhPjeJRGpox6rPcHE5Np6tFx3", txType: core.CoinBase, flags: FlagNotifyConfirmed | FlagNotifyInSyncing, } unconfirmedListener := &TxListener{ - address: "Ef2bDPwcUKguteJutJQCmjX2wgHVfkJ2Wq", + t: t, + service: service, + address: "8ZNizBf4KhhPjeJRGpox6rPcHE5Np6tFx3", txType: core.TransferAsset, flags: 0, } @@ -174,6 +200,29 @@ out: select { case <-syncTicker.C: + best, err := service.headers.GetBest() + if !assert.NoError(t, err) { + t.FailNow() + } + + height := rand.Int31n(int32(best.Height)) + t.Logf("GetTransactionIds from height %d", height) + + txIds, err := service.GetTransactionIds(uint32(height)) + if !assert.NoError(t, err) { + t.FailNow() + } + + for _, txId := range txIds { + tx, err := service.GetTransaction(txId) + if !assert.NotNil(t, tx) { + t.FailNow() + } + if !assert.NoError(t, err) { + t.FailNow() + } + } + if service.IService.IsCurrent() { // Clear test data err := service.ClearData() diff --git a/interface/spvservice.go b/interface/spvservice.go index dfca0c6..f89778b 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -260,9 +260,9 @@ func (s *spvservice) BlockCommitted(block *util.Block) { bloom.MerkleProof{ BlockHash: header.Hash(), Height: header.Height, - Transactions: block.NumTxs, - Hashes: block.Hashes, - Flags: block.Flags, + Transactions: header.NumTxs, + Hashes: header.Hashes, + Flags: header.Flags, }, tx, block.Height-item.Height, diff --git a/interface/store/databatch.go b/interface/store/databatch.go index 60f7e52..b947e9f 100644 --- a/interface/store/databatch.go +++ b/interface/store/databatch.go @@ -42,7 +42,7 @@ func (b *dataBatch) DelAll(height uint32) error { defer b.mutex.Unlock() var key [4]byte - binary.LittleEndian.PutUint32(key[:], height) + binary.BigEndian.PutUint32(key[:], height) data := b.boltTx.Bucket(BKTHeightTxs).Get(key[:]) var txMap = make(map[common.Uint256]uint32) diff --git a/interface/store/txs.go b/interface/store/txs.go index cf55e6e..fec53ff 100644 --- a/interface/store/txs.go +++ b/interface/store/txs.go @@ -3,7 +3,6 @@ package store import ( "bytes" "encoding/binary" - "encoding/gob" "sync" "github.com/elastos/Elastos.ELA.SPV/util" @@ -60,21 +59,27 @@ func (t *txs) Put(txn *util.Tx) (err error) { } var key [4]byte - binary.LittleEndian.PutUint32(key[:], txn.Height) + binary.BigEndian.PutUint32(key[:], txn.Height) data := tx.Bucket(BKTHeightTxs).Get(key[:]) - var txMap = make(map[common.Uint256]uint32) - gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) + data = putTxId(data, &txn.Hash) - txMap[txn.Hash] = txn.Height + return tx.Bucket(BKTHeightTxs).Put(key[:], data) + }) +} - buf = new(bytes.Buffer) - if err = gob.NewEncoder(buf).Encode(txMap); err != nil { - return err - } +func putTxId(data []byte, txId *common.Uint256) []byte { + // Get tx count + var count uint16 + if len(data) == 0 { + data = append(data, 0, 0) + } else { + count = binary.BigEndian.Uint16(data[0:2]) + } - return tx.Bucket(BKTHeightTxs).Put(key[:], buf.Bytes()) - }) + data = append(data, txId[:]...) + binary.BigEndian.PutUint16(data[0:2], count+1) + return data } func (t *txs) Get(hash *common.Uint256) (txn *util.Tx, err error) { @@ -96,27 +101,36 @@ func (t *txs) GetIds(height uint32) (txIds []*common.Uint256, err error) { err = t.View(func(tx *bolt.Tx) error { var key [4]byte - binary.LittleEndian.PutUint32(key[:], height) + binary.BigEndian.PutUint32(key[:], height) data := tx.Bucket(BKTHeightTxs).Get(key[:]) - var txMap = make(map[common.Uint256]uint32) - err = gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) - if err != nil { - return err - } + txIds = getTxIds(data) - txIds = make([]*common.Uint256, 0, len(txMap)) - for hash := range txMap { - var txId common.Uint256 - copy(txId[:], hash[:]) - txIds = append(txIds, &txId) - } return nil }) return txIds, err } +func getTxIds(data []byte) (txIds []*common.Uint256) { + // Get tx count + var count uint16 + if len(data) == 0 { + return nil + } else { + count = binary.BigEndian.Uint16(data[0:2]) + } + + data = data[2:] + for i := uint16(0); i < count; i++ { + var txId common.Uint256 + copy(txId[:], data[i*common.UINT256SIZE : (i+1)*common.UINT256SIZE]) + txIds = append(txIds, &txId) + } + + return txIds +} + func (t *txs) GetAll() (txs []*util.Tx, err error) { t.RLock() defer t.RUnlock() @@ -149,23 +163,37 @@ func (t *txs) Del(txId *common.Uint256) (err error) { } var key [4]byte - binary.LittleEndian.PutUint32(key[:], txn.Height) + binary.BigEndian.PutUint32(key[:], txn.Height) data = tx.Bucket(BKTHeightTxs).Get(key[:]) - var txMap = make(map[common.Uint256]uint32) - err = gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) - if err != nil { - return err - } - delete(txMap, *txId) + data = delTxId(data, &txn.Hash) - var buf = new(bytes.Buffer) - if err = gob.NewEncoder(buf).Encode(txMap); err != nil { - return err + return tx.Bucket(BKTHeightTxs).Put(key[:], data) + }) +} + +func delTxId(data []byte, hash *common.Uint256) []byte{ + // Get tx count + var count uint16 + if len(data) == 0 { + return nil + } else { + count = binary.BigEndian.Uint16(data[0:2]) + } + + data = data[2:] + for i := uint16(0); i < count; i++ { + var txId common.Uint256 + copy(txId[:],data[i*common.UINT256SIZE : (i+1)*common.UINT256SIZE]) + if txId.IsEqual(*hash) { + data = append(data[0:i*common.UINT256SIZE], data[(i+1)*common.UINT256SIZE:]...) + break } + } + var buf [2]byte + binary.BigEndian.PutUint16(buf[:], count-1) - return tx.Bucket(BKTHeightTxs).Put(key[:], buf.Bytes()) - }) + return append(buf[:], data...) } func (t *txs) Batch() TxsBatch { diff --git a/interface/store/txs_test.go b/interface/store/txs_test.go new file mode 100644 index 0000000..bf9781c --- /dev/null +++ b/interface/store/txs_test.go @@ -0,0 +1,46 @@ +package store + +import ( + "crypto/rand" + "encoding/binary" + "fmt" + math "math/rand" + "testing" + + "github.com/elastos/Elastos.ELA.Utility/common" +) + +func TestTxIds(t *testing.T) { + + var data []byte + + var txIds []*common.Uint256 + for i := 0; i < 10; i++ { + var txId common.Uint256 + rand.Read(txId[:]) + txIds = append(txIds, &txId) + fmt.Println(txId) + } + + data = putTxId(data, txIds[0]) + fmt.Println(data) + + fmt.Println(getTxIds(data)) + + data = delTxId(data, txIds[0]) + fmt.Println(data) + + for _, txId := range txIds { + data = putTxId(data, txId) + fmt.Println(getTxIds(data)) + fmt.Println() + } + + for txIds := getTxIds(data); len(txIds) > 0; txIds = getTxIds(data) { + count := binary.BigEndian.Uint16(data[:2]) + txId := txIds[math.Intn(int(count))] + data = delTxId(data, txId) + fmt.Println(txIds) + fmt.Println() + } +} diff --git a/interface/store/txsbatch.go b/interface/store/txsbatch.go index 11e5ee1..967c03a 100644 --- a/interface/store/txsbatch.go +++ b/interface/store/txsbatch.go @@ -65,7 +65,7 @@ func (b *txsBatch) DelAll(height uint32) error { defer b.Unlock() var key [4]byte - binary.LittleEndian.PutUint32(key[:], height) + binary.BigEndian.PutUint32(key[:], height) data := b.Tx.Bucket(BKTHeightTxs).Get(key[:]) var txMap = make(map[common.Uint256]uint32) @@ -95,7 +95,7 @@ func (b *txsBatch) Commit() error { groups := groupByHeight(b.addTxs) for height, txs := range groups { var key [4]byte - binary.LittleEndian.PutUint32(key[:], height) + binary.BigEndian.PutUint32(key[:], height) data := index.Get(key[:]) var txMap = make(map[common.Uint256]uint32) @@ -120,7 +120,7 @@ func (b *txsBatch) Commit() error { groups := groupByHeight(b.delTxs) for height, txs := range groups { var key [4]byte - binary.LittleEndian.PutUint32(key[:], height) + binary.BigEndian.PutUint32(key[:], height) data := index.Get(key[:]) var txMap = make(map[common.Uint256]uint32) From 195540b80a6a092e23d9a9d0a90d4ba1afa6e5d7 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 29 Oct 2018 18:11:09 +0800 Subject: [PATCH 52/73] remove MinPeersForSync parameter from configurations --- interface/interface.go | 3 --- interface/spvservice.go | 1 - sdk/interface.go | 3 --- sdk/service.go | 13 ------------- spvwallet.go | 2 -- sync/config.go | 11 ++++------- sync/manager.go | 5 ----- 7 files changed, 4 insertions(+), 34 deletions(-) diff --git a/interface/interface.go b/interface/interface.go index 933a444..7f08da0 100644 --- a/interface/interface.go +++ b/interface/interface.go @@ -23,9 +23,6 @@ type Config struct { // NodePort is the default port for public peers provide services. DefaultPort uint16 - // The min candidate peers count to start syncing progress. - MinPeersForSync int - // The minimum target outbound connections. MinOutbound int diff --git a/interface/spvservice.go b/interface/spvservice.go index f89778b..d6c426e 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -59,7 +59,6 @@ func newSpvService(cfg *Config) (*spvservice, error) { SeedList: cfg.SeedList, DefaultPort: cfg.DefaultPort, MaxPeers: cfg.MaxConnections, - MinPeersForSync: cfg.MinPeersForSync, GenesisHeader: GenesisHeader(foundation), ChainStore: chainStore, NewTransaction: func() util.Transaction { diff --git a/sdk/interface.go b/sdk/interface.go index ed63569..5f198df 100644 --- a/sdk/interface.go +++ b/sdk/interface.go @@ -68,9 +68,6 @@ type Config struct { // The max peer connections. MaxPeers int - // The min candidate peers count to start syncing progress. - MinPeersForSync int - // GenesisHeader is the GenesisHeader util.BlockHeader diff --git a/sdk/service.go b/sdk/service.go index de0f97f..652b287 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -74,21 +74,13 @@ func newService(cfg *Config) (*service, error) { } var maxPeers int - var minPeersForSync int if cfg.MaxPeers > 0 { maxPeers = cfg.MaxPeers } - if cfg.MinPeersForSync > 0 { - minPeersForSync = cfg.MinPeersForSync - } - if cfg.MinPeersForSync > cfg.MaxPeers { - minPeersForSync = cfg.MaxPeers - } // Create sync manager instance. syncCfg := sync.NewDefaultConfig(chain, service.updateFilter) syncCfg.MaxPeers = maxPeers - syncCfg.MinPeersForSync = minPeersForSync if cfg.StateNotifier != nil { syncCfg.TransactionAnnounce = cfg.StateNotifier.TransactionAnnounce } @@ -375,11 +367,6 @@ cleanup: } func (s *service) SendTransaction(tx util.Transaction) error { - peersCount := s.IServer.ConnectedCount() - if peersCount < int32(s.cfg.MinPeersForSync) { - return fmt.Errorf("connected peers %d not enough for sending transactions", peersCount) - } - if !s.IsCurrent() { return fmt.Errorf("spv service did not sync to current") } diff --git a/spvwallet.go b/spvwallet.go index a75989a..6adcfe3 100644 --- a/spvwallet.go +++ b/spvwallet.go @@ -22,7 +22,6 @@ import ( const ( MaxPeers = 12 - MinPeersForSync = 2 ) var ErrInvalidParameter = fmt.Errorf("invalide parameter") @@ -306,7 +305,6 @@ func NewWallet() (*spvwallet, error) { SeedList: config.SeedList, DefaultPort: config.NodePort, MaxPeers: MaxPeers, - MinPeersForSync: MinPeersForSync, GenesisHeader: GenesisHeader(), ChainStore: chainStore, NewTransaction: newTransaction, diff --git a/sync/config.go b/sync/config.go index 57b5f6f..c0dd495 100644 --- a/sync/config.go +++ b/sync/config.go @@ -7,15 +7,13 @@ import ( ) const ( - DefaultMinPeersForSync = 3 - DefaultMaxPeers = 125 + defaultMaxPeers = 125 ) // Config is a configuration struct used to initialize a new SyncManager. type Config struct { Chain *blockchain.BlockChain - MinPeersForSync int MaxPeers int UpdateFilter func() *bloom.Filter @@ -25,9 +23,8 @@ type Config struct { func NewDefaultConfig(chain *blockchain.BlockChain, updateFilter func() *bloom.Filter) *Config { return &Config{ - Chain: chain, - MinPeersForSync: DefaultMinPeersForSync, - MaxPeers: DefaultMaxPeers, - UpdateFilter: updateFilter, + Chain: chain, + MaxPeers: defaultMaxPeers, + UpdateFilter: updateFilter, } } diff --git a/sync/manager.go b/sync/manager.go index 5a32f77..793e07d 100644 --- a/sync/manager.go +++ b/sync/manager.go @@ -148,11 +148,6 @@ func (sm *SyncManager) current() bool { // simply returns. It also examines the candidates for any which are no longer // candidates and removes them as needed. func (sm *SyncManager) startSync() { - // Return if sync candidates less then MinPeersForSync. - if len(sm.getSyncCandidates()) < sm.cfg.MinPeersForSync { - return - } - // Return now if we're already syncing. if sm.syncPeer != nil { return From 4de0fff4d68d7fd3ccc41124719616f5bc6ddf2f Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 30 Oct 2018 20:04:35 +0800 Subject: [PATCH 53/73] remove trace logs from project --- config.go | 39 ++++++++++++++++++++++++++++++++++++- interface/interface.go | 2 +- interface/interface_test.go | 12 ++++++------ main.go | 2 +- peer/peer.go | 5 +---- sdk/service.go | 2 -- spvwallet.go | 2 +- sync/manager.go | 2 -- 8 files changed, 48 insertions(+), 18 deletions(-) diff --git a/config.go b/config.go index 9cfbc03..c4205d5 100644 --- a/config.go +++ b/config.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "os" "github.com/elastos/Elastos.ELA.Utility/common" @@ -19,7 +20,7 @@ var config = loadConfig() type Config struct { Magic uint32 SeedList []string - NodePort uint16 // node port for public peers to provide services. + DefaultPort uint16 // node port for public peers to provide services. Foundation string PrintLevel int MaxLogsSize int64 @@ -45,6 +46,8 @@ func loadConfig() *Config { os.Exit(-1) } + c.SeedList = normalizeAddresses(c.SeedList, fmt.Sprint(c.DefaultPort)) + if c.Foundation == "" { c.Foundation = "8VYXVxKKSAxkmRrfmGpQR2Kc66XhG6m3ta" } @@ -57,3 +60,37 @@ func loadConfig() *Config { return &c } + +// removeDuplicateAddresses returns a new slice with all duplicate entries in +// addrs removed. +func removeDuplicateAddresses(addrs []string) []string { + result := make([]string, 0, len(addrs)) + seen := map[string]struct{}{} + for _, val := range addrs { + if _, ok := seen[val]; !ok { + result = append(result, val) + seen[val] = struct{}{} + } + } + return result +} + +// normalizeAddress returns addr with the passed default port appended if +// there is not already a port specified. +func normalizeAddress(addr, defaultPort string) string { + _, _, err := net.SplitHostPort(addr) + if err != nil { + return net.JoinHostPort(addr, defaultPort) + } + return addr +} + +// normalizeAddresses returns a new slice with all the passed peer addresses +// normalized with the given default port, and all duplicates removed. +func normalizeAddresses(addrs []string, defaultPort string) []string { + for i, addr := range addrs { + addrs[i] = normalizeAddress(addr, defaultPort) + } + + return removeDuplicateAddresses(addrs) +} diff --git a/interface/interface.go b/interface/interface.go index 7f08da0..9805bfa 100644 --- a/interface/interface.go +++ b/interface/interface.go @@ -20,7 +20,7 @@ type Config struct { // The public seed peers addresses. SeedList []string - // NodePort is the default port for public peers provide services. + // DefaultPort is the default port for public peers provide services. DefaultPort uint16 // The minimum target outbound connections. diff --git a/interface/interface_test.go b/interface/interface_test.go index cb03ce3..09e29ee 100644 --- a/interface/interface_test.go +++ b/interface/interface_test.go @@ -128,12 +128,12 @@ func TestNewSPVService(t *testing.T) { backend := elalog.NewBackend(os.Stdout, elalog.Lshortfile) admrlog := backend.Logger("ADMR", elalog.LevelOff) cmgrlog := backend.Logger("CMGR", elalog.LevelOff) - bcdblog := backend.Logger("BCDB", elalog.LevelTrace) - synclog := backend.Logger("SYNC", elalog.LevelTrace) - peerlog := backend.Logger("PEER", elalog.LevelTrace) - spvslog := backend.Logger("SPVS", elalog.LevelTrace) - srvrlog := backend.Logger("SRVR", elalog.LevelTrace) - rpcslog := backend.Logger("RPCS", elalog.LevelTrace) + bcdblog := backend.Logger("BCDB", elalog.LevelDebug) + synclog := backend.Logger("SYNC", elalog.LevelDebug) + peerlog := backend.Logger("PEER", elalog.LevelDebug) + spvslog := backend.Logger("SPVS", elalog.LevelDebug) + srvrlog := backend.Logger("SRVR", elalog.LevelDebug) + rpcslog := backend.Logger("RPCS", elalog.LevelDebug) addrmgr.UseLogger(admrlog) connmgr.UseLogger(cmgrlog) diff --git a/main.go b/main.go index fa70298..edc7cac 100644 --- a/main.go +++ b/main.go @@ -19,7 +19,7 @@ func main() { signal.Notify(c, os.Interrupt) go func() { for range c { - waltlog.Trace("Wallet shutting down...") + waltlog.Debug("Wallet shutting down...") w.Stop() stop <- 1 } diff --git a/peer/peer.go b/peer/peer.go index 0b1158e..a50a1a8 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -232,7 +232,6 @@ cleanup: break cleanup } } - log.Tracef("Peer stall handler done for %v", p) } // blockHandler handles the downloading of merkleblock and the transactions @@ -355,7 +354,6 @@ cleanup: break cleanup } } - log.Tracef("Peer block handler done for %v", p) } // queueHandler handles the queuing of outgoing data for the peer. This runs as @@ -431,7 +429,6 @@ cleanup: break cleanup } } - log.Tracef("Peer queue handler done for %s", p) } // PushGetBlocksMsg sends a getblocks message for the provided block locator @@ -454,7 +451,7 @@ func (p *Peer) PushGetBlocksMsg(locator []*common.Uint256, stopHash *common.Uint p.prevGetBlocksMtx.Unlock() if isDuplicate { - log.Tracef("Filtering duplicate [getblocks] with begin "+ + log.Debugf("Filtering duplicate [getblocks] with begin "+ "hash %v, stop hash %v", beginHash, stopHash) return nil } diff --git a/sdk/service.go b/sdk/service.go index 652b287..60bca57 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -222,7 +222,6 @@ cleanup: break cleanup } } - log.Trace("Service peers handler done") } // txHandler handles transaction messages like send transaction, transaction inv @@ -363,7 +362,6 @@ cleanup: break cleanup } } - log.Trace("Service transaction handler done") } func (s *service) SendTransaction(tx util.Transaction) error { diff --git a/spvwallet.go b/spvwallet.go index 6adcfe3..2127364 100644 --- a/spvwallet.go +++ b/spvwallet.go @@ -303,7 +303,7 @@ func NewWallet() (*spvwallet, error) { &sdk.Config{ Magic: config.Magic, SeedList: config.SeedList, - DefaultPort: config.NodePort, + DefaultPort: config.DefaultPort, MaxPeers: MaxPeers, GenesisHeader: GenesisHeader(), ChainStore: chainStore, diff --git a/sync/manager.go b/sync/manager.go index 793e07d..ee143be 100644 --- a/sync/manager.go +++ b/sync/manager.go @@ -649,7 +649,6 @@ cleanup: break cleanup } } - log.Trace("Block handler done") } // NewPeer informs the sync manager of a newly active peer. @@ -715,7 +714,6 @@ func (sm *SyncManager) Start() { return } - log.Trace("Starting sync manager") go sm.blockHandler() } From de7434581d2d42b7a50b5d74db2569b600fa3704 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 8 Nov 2018 20:30:39 +0800 Subject: [PATCH 54/73] add DataDir into SPV service config --- sdk/interface.go | 3 +++ sdk/service.go | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/sdk/interface.go b/sdk/interface.go index 5f198df..b249fc2 100644 --- a/sdk/interface.go +++ b/sdk/interface.go @@ -56,6 +56,9 @@ type StateNotifier interface { // Config is the configuration settings to the SPV service. type Config struct { + // DataDir is the data path to store peer addresses etc. + DataDir string + // The magic number to indicate which network to access. Magic uint32 diff --git a/sdk/service.go b/sdk/service.go index 60bca57..5b3cfc9 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -2,6 +2,7 @@ package sdk import ( "fmt" + "os" "time" "github.com/elastos/Elastos.ELA.SPV/blockchain" @@ -18,6 +19,7 @@ import ( ) const ( + defaultDataDir = "./" TxExpireTime = time.Hour * 24 TxRebroadcastDuration = time.Minute * 15 ) @@ -91,6 +93,15 @@ func newService(cfg *Config) (*service, error) { service.syncManager = syncManager // Initiate P2P server configuration + dataDir := defaultDataDir + if len(cfg.DataDir) > 0 { + dataDir = cfg.DataDir + } + _, err = os.Stat(dataDir) + if os.IsNotExist(err) { + os.MkdirAll(dataDir, os.ModePerm) + } + serverCfg := server.NewDefaultConfig( cfg.Magic, p2p.EIP001Version, @@ -103,6 +114,7 @@ func newService(cfg *Config) (*service, error) { service.makeEmptyMessage, func() uint64 { return uint64(chain.BestHeight()) }, ) + serverCfg.DataDir = dataDir serverCfg.MaxPeers = maxPeers serverCfg.DisableListen = true serverCfg.DisableRelayTx = true From 49ed3c7b2cbe07b18558e8d7d73dcd3b8b7036a9 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 8 Nov 2018 21:07:19 +0800 Subject: [PATCH 55/73] add DataDir into interface config and store data into data_spv path by default --- interface/interface.go | 3 +++ interface/spvservice.go | 32 ++++++++++++++++++++++++-------- interface/store/datastore.go | 8 +++++--- interface/store/headers.go | 6 ++++-- interface/store/que.go | 8 ++++---- 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/interface/interface.go b/interface/interface.go index 9805bfa..d3cad7a 100644 --- a/interface/interface.go +++ b/interface/interface.go @@ -10,6 +10,9 @@ import ( // SPV service config type Config struct { + // DataDir is the data path to store db files peer addresses etc. + DataDir string + // The magic number that specify which network to connect. Magic uint32 diff --git a/interface/spvservice.go b/interface/spvservice.go index d6c426e..f9ff449 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "errors" "fmt" + "os" "github.com/elastos/Elastos.ELA.SPV/bloom" "github.com/elastos/Elastos.ELA.SPV/database" @@ -17,6 +18,8 @@ import ( "github.com/elastos/Elastos.ELA/core" ) +const defaultDataDir = "./data_spv" + type spvservice struct { sdk.IService headers store.HeaderStore @@ -35,12 +38,24 @@ func newSpvService(cfg *Config) (*spvservice, error) { return nil, fmt.Errorf("Parse foundation address error %s", err) } - headerStore, err := store.NewHeaderStore(newBlockHeader) + dataDir := defaultDataDir + if len(cfg.DataDir) > 0 { + dataDir = cfg.DataDir + } + _, err = os.Stat(dataDir) + if os.IsNotExist(err) { + err := os.MkdirAll(dataDir, os.ModePerm) + if err != nil { + return nil, fmt.Errorf("make data dir failed") + } + } + + headerStore, err := store.NewHeaderStore(dataDir, newBlockHeader) if err != nil { return nil, err } - dataStore, err := store.NewDataStore() + dataStore, err := store.NewDataStore(dataDir) if err != nil { return nil, err } @@ -55,12 +70,13 @@ func newSpvService(cfg *Config) (*spvservice, error) { chainStore := database.NewDefaultChainDB(headerStore, service) serviceCfg := &sdk.Config{ - Magic: cfg.Magic, - SeedList: cfg.SeedList, - DefaultPort: cfg.DefaultPort, - MaxPeers: cfg.MaxConnections, - GenesisHeader: GenesisHeader(foundation), - ChainStore: chainStore, + DataDir: dataDir, + Magic: cfg.Magic, + SeedList: cfg.SeedList, + DefaultPort: cfg.DefaultPort, + MaxPeers: cfg.MaxConnections, + GenesisHeader: GenesisHeader(foundation), + ChainStore: chainStore, NewTransaction: func() util.Transaction { return &core.Transaction{} }, diff --git a/interface/store/datastore.go b/interface/store/datastore.go index 6f81a6e..520505d 100644 --- a/interface/store/datastore.go +++ b/interface/store/datastore.go @@ -2,6 +2,7 @@ package store import ( "github.com/boltdb/bolt" + "path/filepath" "sync" ) @@ -17,8 +18,9 @@ type dataStore struct { que *que } -func NewDataStore() (*dataStore, error) { - db, err := bolt.Open("data_store.bin", 0644, &bolt.Options{InitialMmapSize: 5000000}) +func NewDataStore(dataDir string) (*dataStore, error) { + db, err := bolt.Open(filepath.Join(dataDir, "data_store.bin"), 0644, + &bolt.Options{InitialMmapSize: 5000000}) if err != nil { return nil, err } @@ -41,7 +43,7 @@ func NewDataStore() (*dataStore, error) { return nil, err } - store.que, err = NewQue() + store.que, err = NewQue(dataDir) if err != nil { return nil, err } diff --git a/interface/store/headers.go b/interface/store/headers.go index 5e70149..904cc7b 100644 --- a/interface/store/headers.go +++ b/interface/store/headers.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "math/big" + "path/filepath" "sync" "github.com/elastos/Elastos.ELA.SPV/util" @@ -32,8 +33,9 @@ type headers struct { newHeader func() util.BlockHeader } -func NewHeaderStore(newHeader func() util.BlockHeader) (*headers, error) { - db, err := bolt.Open("headers.bin", 0644, &bolt.Options{InitialMmapSize: 5000000}) +func NewHeaderStore(dataDir string, newHeader func() util.BlockHeader) (*headers, error) { + db, err := bolt.Open(filepath.Join(dataDir, "headers.bin"), 0644, + &bolt.Options{InitialMmapSize: 5000000}) if err != nil { return nil, err } diff --git a/interface/store/que.go b/interface/store/que.go index dbce964..ab18221 100644 --- a/interface/store/que.go +++ b/interface/store/que.go @@ -3,6 +3,7 @@ package store import ( "database/sql" "fmt" + "path/filepath" "sync" "github.com/elastos/Elastos.ELA.Utility/common" @@ -10,7 +11,6 @@ import ( const ( DriverName = "sqlite3" - DBName = "./queue.db" CreateQueueDB = `CREATE TABLE IF NOT EXISTS Queue( NotifyId BLOB NOT NULL, @@ -29,10 +29,10 @@ type que struct { *sql.DB } -func NewQue() (*que, error) { - db, err := sql.Open(DriverName, DBName) +func NewQue(dataDir string) (*que, error) { + db, err := sql.Open(DriverName, filepath.Join(dataDir, "queue.db")) if err != nil { - fmt.Println("Open sqlite db error:", err) + fmt.Println("Open queue db error:", err) return nil, err } From f38539a0bd9d65447380bdfd5f7777dcbac059e8 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Fri, 9 Nov 2018 18:15:46 +0800 Subject: [PATCH 56/73] handle interrupt signals with Utility signal helper --- interface/interface_test.go | 10 ++++++++-- main.go | 23 ++++++++--------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/interface/interface_test.go b/interface/interface_test.go index 09e29ee..4b4bbd2 100644 --- a/interface/interface_test.go +++ b/interface/interface_test.go @@ -2,6 +2,7 @@ package _interface import ( "fmt" + "github.com/elastos/Elastos.ELA.Utility/signal" "math/rand" "os" "testing" @@ -51,7 +52,6 @@ func (l *TxListener) Notify(id common.Uint256, proof bloom.MerkleProof, tx core. l.t.FailNow() } - txIds, err := l.service.GetTransactionIds(proof.Height) if !assert.NotNil(l.t, tx) { l.t.FailNow() @@ -125,6 +125,8 @@ func TestGetListenerKey(t *testing.T) { } func TestNewSPVService(t *testing.T) { + interrupt := signal.NewInterrupt() + backend := elalog.NewBackend(os.Stdout, elalog.Lshortfile) admrlog := backend.Logger("ADMR", elalog.LevelOff) cmgrlog := backend.Logger("CMGR", elalog.LevelOff) @@ -198,6 +200,9 @@ func TestNewSPVService(t *testing.T) { out: for { select { + case <-interrupt.C: + break out + case <-syncTicker.C: best, err := service.headers.GetBest() @@ -230,11 +235,12 @@ out: t.Errorf("service clear data error %s", err) } - service.Stop() t.Log("successful synchronized to current") break out } } } + + service.Stop() } diff --git a/main.go b/main.go index edc7cac..319de2d 100644 --- a/main.go +++ b/main.go @@ -2,30 +2,23 @@ package main import ( "os" - "os/signal" + + "github.com/elastos/Elastos.ELA.Utility/signal" ) func main() { - // Initiate SPV service + // Listen interrupt signals. + interrupt := signal.NewInterrupt() + + // Create the SPV wallet instance. w, err := NewWallet() if err != nil { waltlog.Error("Initiate SPV service failed,", err) os.Exit(0) } - - // Handle interrupt signal - stop := make(chan int, 1) - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - go func() { - for range c { - waltlog.Debug("Wallet shutting down...") - w.Stop() - stop <- 1 - } - }() + defer w.Stop() w.Start() - <-stop + <-interrupt.C } From a035f95dcd15be724dd9305cc5ae468eba6113eb Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 22 Nov 2018 17:25:59 +0800 Subject: [PATCH 57/73] use levelDB instead of boltDB to store headers --- .gitignore | 3 +- config.json.back | 11 -- glide.yaml | 2 +- interface/interface_test.go | 16 ++- interface/store/addrs.go | 65 +++++------- interface/store/databatch.go | 65 ++++-------- interface/store/datastore.go | 66 ++++++------ interface/store/headers.go | 146 ++++++++++---------------- interface/store/ops.go | 84 ++++++--------- interface/store/opsbatch.go | 20 +++- interface/store/que.go | 5 +- interface/store/txs.go | 175 ++++++++++++++----------------- interface/store/txsbatch.go | 90 +++++----------- wallet/store/headers/cache.go | 2 +- wallet/store/headers/database.go | 130 +++++++++-------------- 15 files changed, 360 insertions(+), 520 deletions(-) delete mode 100644 config.json.back diff --git a/.gitignore b/.gitignore index 6ce5de9..93b87f8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ idea/ build/ *.patch .idea/ -GoOnchain.iml .DS_Store *.orig *.rej @@ -22,5 +21,7 @@ Log *.lock *.json vendor/ +HEADER/ +DATA/ ela-wallet service diff --git a/config.json.back b/config.json.back deleted file mode 100644 index f366f89..0000000 --- a/config.json.back +++ /dev/null @@ -1,11 +0,0 @@ -{ - "Magic": 20180627, - "PrintLevel": 0, - "RPCPort": 20477, - "Foundation": "8ZNizBf4KhhPjeJRGpox6rPcHE5Np6tFx3", - "SeedList": [ - "node-regtest-002.elastos.org:22866", - "node-regtest-003.elastos.org:22866", - "node-regtest-004.elastos.org:22866" - ] -} \ No newline at end of file diff --git a/glide.yaml b/glide.yaml index fc21b9d..a89f515 100644 --- a/glide.yaml +++ b/glide.yaml @@ -5,7 +5,7 @@ import: - package: golang.org/x/sys repo: https://github.com/golang/sys - package: github.com/howeyc/gopass -- package: github.com/boltdb/bolt +- package: github.com/syndtr/goleveldb/leveldb - package: github.com/cevaris/ordered_map - package: github.com/elastos/Elastos.ELA.Utility version: release_v0.1.1 diff --git a/interface/interface_test.go b/interface/interface_test.go index 4b4bbd2..13b0063 100644 --- a/interface/interface_test.go +++ b/interface/interface_test.go @@ -2,7 +2,6 @@ package _interface import ( "fmt" - "github.com/elastos/Elastos.ELA.Utility/signal" "math/rand" "os" "testing" @@ -21,6 +20,7 @@ import ( "github.com/elastos/Elastos.ELA.Utility/p2p/addrmgr" "github.com/elastos/Elastos.ELA.Utility/p2p/connmgr" "github.com/elastos/Elastos.ELA.Utility/p2p/server" + "github.com/elastos/Elastos.ELA.Utility/signal" "github.com/elastos/Elastos.ELA/core" "github.com/stretchr/testify/assert" ) @@ -168,7 +168,7 @@ func TestNewSPVService(t *testing.T) { service, err := newSpvService(config) if err != nil { - t.Error("NewSPVService error %s", err.Error()) + t.Errorf("NewSPVService error %s", err.Error()) } confirmedListener := &TxListener{ @@ -210,10 +210,16 @@ out: t.FailNow() } - height := rand.Int31n(int32(best.Height)) - t.Logf("GetTransactionIds from height %d", height) + height := uint32(rand.Int31n(int32(best.Height))) + + t.Logf("GetBlock from height %d", height) + _, err = service.headers.GetByHeight(height) + if !assert.NoError(t, err) { + t.FailNow() + } - txIds, err := service.GetTransactionIds(uint32(height)) + t.Logf("GetTransactionIds from height %d", height) + txIds, err := service.GetTransactionIds(height) if !assert.NoError(t, err) { t.FailNow() } diff --git a/interface/store/addrs.go b/interface/store/addrs.go index 5cce77e..8ca6029 100644 --- a/interface/store/addrs.go +++ b/interface/store/addrs.go @@ -6,35 +6,25 @@ import ( "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.Utility/common" - - "github.com/boltdb/bolt" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" ) var ( - BKTAddrs = []byte("Addrs") + BKTAddrs = []byte("A") ) // Ensure addrs implement Addrs interface. var _ Addrs = (*addrs)(nil) type addrs struct { - *sync.RWMutex - *bolt.DB + sync.RWMutex + db *leveldb.DB filter *sdk.AddrFilter } -func NewAddrs(db *bolt.DB) (*addrs, error) { - store := new(addrs) - store.RWMutex = new(sync.RWMutex) - store.DB = db - - db.Update(func(btx *bolt.Tx) error { - _, err := btx.CreateBucketIfNotExists(BKTAddrs) - if err != nil { - return err - } - return nil - }) +func NewAddrs(db *leveldb.DB) (*addrs, error) { + store := addrs{db: db} addrs, err := store.getAll() if err != nil { @@ -42,7 +32,7 @@ func NewAddrs(db *bolt.DB) (*addrs, error) { } store.filter = sdk.NewAddrFilter(addrs) - return store, nil + return &store, nil } func (a *addrs) GetFilter() *sdk.AddrFilter { @@ -60,39 +50,40 @@ func (a *addrs) Put(addr *common.Uint168) error { } a.filter.AddAddr(addr) - return a.Update(func(tx *bolt.Tx) error { - return tx.Bucket(BKTAddrs).Put(addr.Bytes(), addr.Bytes()) - }) + return a.db.Put(toKey(BKTAddrs, addr[:]...), addr[:], nil) } func (a *addrs) GetAll() []*common.Uint168 { a.RLock() defer a.RUnlock() - return a.filter.GetAddrs() } func (a *addrs) getAll() (addrs []*common.Uint168, err error) { - err = a.View(func(tx *bolt.Tx) error { - return tx.Bucket(BKTAddrs).ForEach(func(k, v []byte) error { - addr, err := common.Uint168FromBytes(v) - if err != nil { - return err - } - addrs = append(addrs, addr) - return nil - }) - }) - - return addrs, err + it := a.db.NewIterator(util.BytesPrefix(BKTAddrs), nil) + defer it.Release() + for it.Next() { + addr, err := common.Uint168FromBytes(it.Value()) + if err != nil { + return nil, err + } + addrs = append(addrs, addr) + } + + return addrs, it.Error() } func (a *addrs) Clear() error { a.Lock() defer a.Unlock() - return a.DB.Update(func(tx *bolt.Tx) error { - return tx.DeleteBucket(BKTAddrs) - }) + + it := a.db.NewIterator(util.BytesPrefix(BKTAddrs), nil) + defer it.Release() + batch := new(leveldb.Batch) + for it.Next() { + batch.Delete(it.Key()) + } + return a.db.Write(batch, nil) } func (a *addrs) Close() error { diff --git a/interface/store/databatch.go b/interface/store/databatch.go index b947e9f..1db6baf 100644 --- a/interface/store/databatch.go +++ b/interface/store/databatch.go @@ -4,62 +4,53 @@ import ( "bytes" "database/sql" "encoding/binary" - "encoding/gob" "sync" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/boltdb/bolt" - "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" + "github.com/syndtr/goleveldb/leveldb" ) // Ensure dataBatch implement DataBatch interface. var _ DataBatch = (*dataBatch)(nil) type dataBatch struct { - mutex sync.Mutex - boltTx *bolt.Tx - sqlTx *sql.Tx + mutex sync.Mutex + *leveldb.DB + *leveldb.Batch + sqlTx *sql.Tx } func (b *dataBatch) Txs() TxsBatch { - return &txsBatch{Tx: b.boltTx} + return &txsBatch{Batch: b.Batch} } func (b *dataBatch) Ops() OpsBatch { - return &opsBatch{Tx: b.boltTx} + return &opsBatch{Batch: b.Batch} } func (b *dataBatch) Que() QueBatch { return &queBatch{Tx: b.sqlTx} } -// Delete all transactions, ops, queued items on -// the given height. +// Delete all transactions, ops, queued items on the given height. func (b *dataBatch) DelAll(height uint32) error { b.mutex.Lock() defer b.mutex.Unlock() var key [4]byte binary.BigEndian.PutUint32(key[:], height) - data := b.boltTx.Bucket(BKTHeightTxs).Get(key[:]) - - var txMap = make(map[common.Uint256]uint32) - err := gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) - if err != nil { - return err - } - - txsBucket := b.boltTx.Bucket(BKTTxs) - opsBucket := b.boltTx.Bucket(BKTOps) - for txId := range txMap { + data, _ := b.DB.Get(toKey(BKTHeightTxs, key[:]...), nil) + for _, txId := range getTxIds(data) { var utx util.Tx - data := txsBucket.Get(txId.Bytes()) - err := utx.Deserialize(bytes.NewReader(data)) + data, err := b.DB.Get(toKey(BKTTxs, txId.Bytes()...), nil) if err != nil { return err } + if err := utx.Deserialize(bytes.NewReader(data)); err != nil { + return err + } var tx core.Transaction err = tx.Deserialize(bytes.NewReader(utx.RawData)) @@ -68,25 +59,22 @@ func (b *dataBatch) DelAll(height uint32) error { } for index := range tx.Outputs { - outpoint := core.NewOutPoint(utx.Hash, uint16(index)).Bytes() - opsBucket.Delete(outpoint) + outpoint := core.NewOutPoint(utx.Hash, uint16(index)) + b.Batch.Delete(toKey(BKTOps, outpoint.Bytes()...)) } - if err := b.boltTx.Bucket(BKTTxs).Delete(txId.Bytes()); err != nil { - return err - } + b.Batch.Delete(toKey(BKTTxs, txId.Bytes()...)) } - err = b.boltTx.Bucket(BKTHeightTxs).Delete(key[:]) - if err != nil { - return err - } + b.Batch.Delete(toKey(BKTHeightTxs, key[:]...)) return b.Que().DelAll(height) } func (b *dataBatch) Commit() error { - if err := b.boltTx.Commit(); err != nil { + defer b.sqlTx.Rollback() + + if err := b.DB.Write(b.Batch, nil); err != nil { return err } @@ -98,13 +86,6 @@ func (b *dataBatch) Commit() error { } func (b *dataBatch) Rollback() error { - if err := b.boltTx.Rollback(); err != nil { - return err - } - - if err := b.sqlTx.Rollback(); err != nil { - return err - } - - return nil + b.Batch.Reset() + return b.sqlTx.Rollback() } diff --git a/interface/store/datastore.go b/interface/store/datastore.go index 520505d..5ef2503 100644 --- a/interface/store/datastore.go +++ b/interface/store/datastore.go @@ -1,17 +1,18 @@ package store import ( - "github.com/boltdb/bolt" "path/filepath" "sync" + + "github.com/syndtr/goleveldb/leveldb" ) // Ensure dataStore implement DataStore interface. var _ DataStore = (*dataStore)(nil) type dataStore struct { - *sync.RWMutex - *bolt.DB + sync.RWMutex + db *leveldb.DB addrs *addrs txs *txs ops *ops @@ -19,36 +20,41 @@ type dataStore struct { } func NewDataStore(dataDir string) (*dataStore, error) { - db, err := bolt.Open(filepath.Join(dataDir, "data_store.bin"), 0644, - &bolt.Options{InitialMmapSize: 5000000}) + db, err := leveldb.OpenFile(filepath.Join(dataDir, "DATA"), nil) + if err != nil { + return nil, err + } if err != nil { return nil, err } - store := new(dataStore) - store.RWMutex = new(sync.RWMutex) - store.DB = db - store.addrs, err = NewAddrs(db) + addrs, err := NewAddrs(db) if err != nil { return nil, err } - store.txs, err = NewTxs(db) + txs, err := NewTxs(db) if err != nil { return nil, err } - store.ops, err = NewOps(db) + ops, err := NewOps(db) if err != nil { return nil, err } - store.que, err = NewQue(dataDir) + que, err := NewQue(dataDir) if err != nil { return nil, err } - return store, nil + return &dataStore{ + db: db, + addrs: addrs, + txs: txs, + ops: ops, + que: que, + }, nil } func (d *dataStore) Addrs() Addrs { @@ -68,18 +74,14 @@ func (d *dataStore) Que() Que { } func (d *dataStore) Batch() DataBatch { - boltTx, err := d.DB.Begin(true) - if err != nil { - panic(err) - } - sqlTx, err := d.que.Begin() if err != nil { panic(err) } return &dataBatch{ - boltTx: boltTx, + DB:d.db, + Batch: new(leveldb.Batch), sqlTx: sqlTx, } } @@ -88,24 +90,26 @@ func (d *dataStore) Clear() error { d.Lock() defer d.Unlock() - return d.Update(func(tx *bolt.Tx) error { - tx.DeleteBucket(BKTAddrs) - tx.DeleteBucket(BKTTxs) - tx.DeleteBucket(BKTHeightTxs) - tx.DeleteBucket(BKTOps) - return nil - }) + d.que.Clear() + + it := d.db.NewIterator(nil, nil) + batch := new(leveldb.Batch) + for it.Next() { + batch.Delete(it.Key()) + } + it.Release() + + return d.db.Write(batch, nil) } // Close db func (d *dataStore) Close() error { - d.addrs.Close() - d.txs.Close() - d.ops.Close() - d.Lock() defer d.Unlock() + d.addrs.Close() + d.txs.Close() + d.ops.Close() d.que.Close() - return d.DB.Close() + return d.db.Close() } diff --git a/interface/store/headers.go b/interface/store/headers.go index 904cc7b..168f6f2 100644 --- a/interface/store/headers.go +++ b/interface/store/headers.go @@ -11,16 +11,15 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/boltdb/bolt" "github.com/cevaris/ordered_map" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/syndtr/goleveldb/leveldb" ) var ( - BKTHeaders = []byte("Headers") - BKTHeightHash = []byte("HeightHash") - BKTChainTip = []byte("ChainTip") - KEYChainTip = []byte("ChainTip") + BKTHeaders = []byte("H") + BKTIndexes = []byte("I") + BKTChainTip = []byte("B") ) // Ensure headers implement database.Headers interface. @@ -28,37 +27,20 @@ var _ HeaderStore = (*headers)(nil) type headers struct { *sync.RWMutex - *bolt.DB + db *leveldb.DB cache *cache newHeader func() util.BlockHeader } func NewHeaderStore(dataDir string, newHeader func() util.BlockHeader) (*headers, error) { - db, err := bolt.Open(filepath.Join(dataDir, "headers.bin"), 0644, - &bolt.Options{InitialMmapSize: 5000000}) + db, err := leveldb.OpenFile(filepath.Join(dataDir, "HEADER"), nil) if err != nil { return nil, err } - db.Update(func(btx *bolt.Tx) error { - _, err := btx.CreateBucketIfNotExists(BKTHeaders) - if err != nil { - return err - } - _, err = btx.CreateBucketIfNotExists(BKTHeightHash) - if err != nil { - return err - } - _, err = btx.CreateBucketIfNotExists(BKTChainTip) - if err != nil { - return err - } - return nil - }) - headers := &headers{ RWMutex: new(sync.RWMutex), - DB: db, + db: db, cache: newCache(100), newHeader: newHeader, } @@ -95,29 +77,29 @@ func (h *headers) Put(header *util.Header, newTip bool) error { if newTip { h.cache.tip = header } - return h.Update(func(tx *bolt.Tx) error { + key := toKey(BKTHeaders, header.Hash().Bytes()...) - bytes, err := header.Serialize() - if err != nil { - return err - } + bytes, err := header.Serialize() + if err != nil { + return err + } - err = tx.Bucket(BKTHeaders).Put(header.Hash().Bytes(), bytes) + err = h.db.Put(key, bytes, nil) + if err != nil { + return err + } + + if newTip { + err = h.db.Put(BKTChainTip, bytes, nil) if err != nil { return err } + } - if newTip { - err = tx.Bucket(BKTChainTip).Put(KEYChainTip, bytes) - if err != nil { - return err - } - } - - var key [4]byte - binary.LittleEndian.PutUint32(key[:], header.Height) - return tx.Bucket(BKTHeightHash).Put(key[:], header.Hash().Bytes()) - }) + var height [4]byte + binary.LittleEndian.PutUint32(height[:], header.Height) + index := toKey(BKTIndexes, height[:]...) + return h.db.Put(index, header.Hash().Bytes(), nil) } func (h *headers) GetPrevious(header *util.Header) (*util.Header, error) { @@ -140,17 +122,7 @@ func (h *headers) Get(hash *common.Uint256) (header *util.Header, err error) { return header, nil } - err = h.View(func(tx *bolt.Tx) error { - - header, err = h.getHeader(tx, BKTHeaders, hash.Bytes()) - if err != nil { - return err - } - - return nil - }) - - return header, err + return h.getHeader(toKey(BKTHeaders, hash.Bytes()...)) } func (h *headers) GetBest() (header *util.Header, err error) { @@ -161,35 +133,23 @@ func (h *headers) GetBest() (header *util.Header, err error) { return h.cache.tip, nil } - err = h.View(func(tx *bolt.Tx) error { - - header, err = h.getHeader(tx, BKTChainTip, KEYChainTip) - if err != nil { - return err - } - - return nil - }) - - return header, err + return h.getHeader(BKTChainTip) } func (h *headers) GetByHeight(height uint32) (header *util.Header, err error) { h.RLock() defer h.RUnlock() - err = h.View(func(tx *bolt.Tx) error { - var key [4]byte - binary.LittleEndian.PutUint32(key[:], height) - hashBytes := tx.Bucket(BKTHeightHash).Get(key[:]) - header, err = h.getHeader(tx, BKTHeaders, hashBytes) - if err != nil { - return err - } - return err - }) + var key [4]byte + binary.LittleEndian.PutUint32(key[:], height) + hashBytes, err := h.db.Get(toKey(BKTIndexes, key[:]...), nil) + if err != nil { + return nil, err + } + + header, err = h.getHeader(toKey(BKTHeaders, hashBytes...)) if err != nil { - return header, fmt.Errorf("header not exist on height %d", height) + return nil, err } return header, err @@ -199,32 +159,32 @@ func (h *headers) Clear() error { h.Lock() defer h.Unlock() - return h.Update(func(tx *bolt.Tx) error { - err := tx.DeleteBucket(BKTHeaders) - if err != nil { - return err - } - - return tx.DeleteBucket(BKTChainTip) - }) + batch := new(leveldb.Batch) + inter := h.db.NewIterator(nil, nil) + for inter.Next() { + batch.Delete(inter.Key()) + } + inter.Release() + return h.db.Write(batch, nil) } // Close db func (h *headers) Close() error { h.Lock() defer h.Unlock() - return h.DB.Close() + return h.db.Close() } -func (h *headers) getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*util.Header, error) { - headerBytes := tx.Bucket(bucket).Get(key) - if headerBytes == nil { - return nil, fmt.Errorf("header %s does not exist in database", hex.EncodeToString(key)) +func (h *headers) getHeader(key []byte) (*util.Header, error) { + data, err := h.db.Get(key, nil) + if err != nil { + return nil, fmt.Errorf("header %s does not exist in database", + hex.EncodeToString(key)) } var header util.Header header.BlockHeader = h.newHeader() - err := header.Deserialize(headerBytes) + err = header.Deserialize(data) if err != nil { return nil, err } @@ -274,3 +234,11 @@ func (cache *cache) Get(hash *common.Uint256) (*util.Header, error) { } return sh.(*util.Header), nil } + +func toKey(bucket []byte, index ...byte) []byte { + return append(bucket, index...) +} + +func subKey(bucket []byte, key []byte) []byte { + return key[len(bucket):] +} diff --git a/interface/store/ops.go b/interface/store/ops.go index e59eebe..93cbcb1 100644 --- a/interface/store/ops.go +++ b/interface/store/ops.go @@ -5,58 +5,44 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/boltdb/bolt" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/syndtr/goleveldb/leveldb" + dbutil "github.com/syndtr/goleveldb/leveldb/util" ) var ( - BKTOps = []byte("Ops") + BKTOps = []byte("O") ) // Ensure ops implement Ops interface. var _ Ops = (*ops)(nil) type ops struct { - *sync.RWMutex - *bolt.DB + sync.RWMutex + db *leveldb.DB } -func NewOps(db *bolt.DB) (*ops, error) { - store := new(ops) - store.RWMutex = new(sync.RWMutex) - store.DB = db - - db.Update(func(btx *bolt.Tx) error { - _, err := btx.CreateBucketIfNotExists(BKTOps) - if err != nil { - return err - } - return nil - }) - - return store, nil +func NewOps(db *leveldb.DB) (*ops, error) { + return &ops{db: db}, nil } -func (o *ops) Put(op *util.OutPoint, addr common.Uint168) (err error) { +func (o *ops) Put(op *util.OutPoint, addr common.Uint168) error { o.Lock() defer o.Unlock() - return o.Update(func(tx *bolt.Tx) error { - return tx.Bucket(BKTOps).Put(op.Bytes(), addr.Bytes()) - }) + return o.db.Put(toKey(BKTOps, op.Bytes()...), addr.Bytes(), nil) } func (o *ops) HaveOp(op *util.OutPoint) (addr *common.Uint168) { o.RLock() defer o.RUnlock() - o.View(func(tx *bolt.Tx) error { - addrBytes := tx.Bucket(BKTOps).Get(op.Bytes()) - var err error - if addr, err = common.Uint168FromBytes(addrBytes); err != nil { - return err - } + addrBytes, err := o.db.Get(toKey(BKTOps, op.Bytes()...), nil) + if err != nil { + return nil + } + if addr, err = common.Uint168FromBytes(addrBytes); err != nil { return nil - }) + } return addr } @@ -64,35 +50,33 @@ func (o *ops) GetAll() (ops []*util.OutPoint, err error) { o.RLock() defer o.RUnlock() - err = o.View(func(tx *bolt.Tx) error { - return tx.Bucket(BKTOps).ForEach(func(k, v []byte) error { - op, err := util.OutPointFromBytes(k) - if err != nil { - return err - } - ops = append(ops, op) - return nil - }) - }) - - return ops, err + it := o.db.NewIterator(dbutil.BytesPrefix(BKTOps), nil) + defer it.Release() + for it.Next() { + op, err := util.OutPointFromBytes(subKey(BKTOps, it.Key())) + if err != nil { + return nil, err + } + ops = append(ops, op) + } + + return ops, it.Error() } func (o *ops) Batch() OpsBatch { - tx, err := o.Begin(true) - if err != nil { - panic(err) - } - - return &opsBatch{Tx: tx} + return &opsBatch{DB: o.db, Batch: new(leveldb.Batch)} } func (o *ops) Clear() error { o.Lock() defer o.Unlock() - return o.DB.Update(func(tx *bolt.Tx) error { - return tx.DeleteBucket(BKTOps) - }) + it := o.db.NewIterator(dbutil.BytesPrefix(BKTOps), nil) + defer it.Release() + batch := new(leveldb.Batch) + for it.Next() { + batch.Delete(it.Key()) + } + return o.db.Write(batch, nil) } func (o *ops) Close() error { diff --git a/interface/store/opsbatch.go b/interface/store/opsbatch.go index c011474..28830c2 100644 --- a/interface/store/opsbatch.go +++ b/interface/store/opsbatch.go @@ -5,8 +5,8 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/boltdb/bolt" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/syndtr/goleveldb/leveldb" ) // Ensure opsBatch implement OpsBatch interface. @@ -14,17 +14,29 @@ var _ OpsBatch = (*opsBatch)(nil) type opsBatch struct { sync.Mutex - *bolt.Tx + *leveldb.DB + *leveldb.Batch } func (b *opsBatch) Put(op *util.OutPoint, addr common.Uint168) error { b.Lock() defer b.Unlock() - return b.Tx.Bucket(BKTOps).Put(op.Bytes(), addr.Bytes()) + b.Batch.Put(toKey(BKTOps, op.Bytes()...), addr[:]) + return nil } func (b *opsBatch) Del(op *util.OutPoint) error { b.Lock() defer b.Unlock() - return b.Tx.Bucket(BKTOps).Delete(op.Bytes()) + b.Batch.Delete(toKey(BKTOps, op.Bytes()...)) + return nil +} + +func (b *opsBatch) Rollback() error { + b.Batch.Reset() + return nil +} + +func (b *opsBatch) Commit() error { + return b.DB.Write(b.Batch, nil) } diff --git a/interface/store/que.go b/interface/store/que.go index ab18221..d0f6da6 100644 --- a/interface/store/que.go +++ b/interface/store/que.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/elastos/Elastos.ELA.Utility/common" + _ "github.com/mattn/go-sqlite3" ) const ( @@ -25,7 +26,7 @@ const ( var _ Que = (*que)(nil) type que struct { - *sync.RWMutex + sync.RWMutex *sql.DB } @@ -40,7 +41,7 @@ func NewQue(dataDir string) (*que, error) { if err != nil { return nil, err } - return &que{RWMutex: new(sync.RWMutex), DB: db}, nil + return &que{DB: db}, nil } // Put a queue item to database diff --git a/interface/store/txs.go b/interface/store/txs.go index fec53ff..ba8e497 100644 --- a/interface/store/txs.go +++ b/interface/store/txs.go @@ -7,65 +7,46 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/boltdb/bolt" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/syndtr/goleveldb/leveldb" + dbutil "github.com/syndtr/goleveldb/leveldb/util" ) var ( - BKTTxs = []byte("Txs") - BKTHeightTxs = []byte("HeightTxs") + BKTTxs = []byte("T") + BKTHeightTxs = []byte("H") ) // Ensure txs implement Txs interface. var _ Txs = (*txs)(nil) type txs struct { - *sync.RWMutex - *bolt.DB + sync.RWMutex + db *leveldb.DB } -func NewTxs(db *bolt.DB) (*txs, error) { - store := new(txs) - store.RWMutex = new(sync.RWMutex) - store.DB = db - - db.Update(func(btx *bolt.Tx) error { - _, err := btx.CreateBucketIfNotExists(BKTTxs) - if err != nil { - return err - } - _, err = btx.CreateBucketIfNotExists(BKTHeightTxs) - if err != nil { - return err - } - return nil - }) - - return store, nil +func NewTxs(db *leveldb.DB) (*txs, error) { + return &txs{db: db}, nil } -func (t *txs) Put(txn *util.Tx) (err error) { +func (t *txs) Put(txn *util.Tx) error { t.Lock() defer t.Unlock() - return t.Update(func(tx *bolt.Tx) error { - buf := new(bytes.Buffer) - if err = txn.Serialize(buf); err != nil { - return err - } - - if err = tx.Bucket(BKTTxs).Put(txn.Hash.Bytes(), buf.Bytes()); err != nil { - return err - } + buf := new(bytes.Buffer) + if err := txn.Serialize(buf); err != nil { + return err + } - var key [4]byte - binary.BigEndian.PutUint32(key[:], txn.Height) - data := tx.Bucket(BKTHeightTxs).Get(key[:]) + batch := new(leveldb.Batch) + batch.Put(toKey(BKTTxs, txn.Hash.Bytes()...), buf.Bytes()) - data = putTxId(data, &txn.Hash) + var key [4]byte + binary.BigEndian.PutUint32(key[:], txn.Height) + data, _ := t.db.Get(toKey(BKTHeightTxs, key[:]...), nil) + batch.Put(toKey(BKTHeightTxs, key[:]...), putTxId(data, &txn.Hash)) - return tx.Bucket(BKTHeightTxs).Put(key[:], data) - }) + return t.db.Write(batch, nil) } func putTxId(data []byte, txId *common.Uint256) []byte { @@ -86,11 +67,15 @@ func (t *txs) Get(hash *common.Uint256) (txn *util.Tx, err error) { t.RLock() defer t.RUnlock() - err = t.View(func(tx *bolt.Tx) error { - data := tx.Bucket(BKTTxs).Get(hash.Bytes()) - txn = new(util.Tx) - return txn.Deserialize(bytes.NewReader(data)) - }) + data, err := t.db.Get(toKey(BKTTxs, hash.Bytes()...), nil) + if err != nil { + return nil, err + } + txn = new(util.Tx) + err = txn.Deserialize(bytes.NewReader(data)) + if err != nil { + return nil, err + } return txn, err } @@ -99,17 +84,10 @@ func (t *txs) GetIds(height uint32) (txIds []*common.Uint256, err error) { t.RLock() defer t.RUnlock() - err = t.View(func(tx *bolt.Tx) error { - var key [4]byte - binary.BigEndian.PutUint32(key[:], height) - data := tx.Bucket(BKTHeightTxs).Get(key[:]) - - txIds = getTxIds(data) - - return nil - }) - - return txIds, err + var key [4]byte + binary.BigEndian.PutUint32(key[:], height) + data, _ := t.db.Get(toKey(BKTHeightTxs, key[:]...), nil) + return getTxIds(data), nil } func getTxIds(data []byte) (txIds []*common.Uint256) { @@ -124,7 +102,7 @@ func getTxIds(data []byte) (txIds []*common.Uint256) { data = data[2:] for i := uint16(0); i < count; i++ { var txId common.Uint256 - copy(txId[:], data[i*common.UINT256SIZE : (i+1)*common.UINT256SIZE]) + copy(txId[:], data[i*common.UINT256SIZE:(i+1)*common.UINT256SIZE]) txIds = append(txIds, &txId) } @@ -135,44 +113,45 @@ func (t *txs) GetAll() (txs []*util.Tx, err error) { t.RLock() defer t.RUnlock() - err = t.View(func(tx *bolt.Tx) error { - return tx.Bucket(BKTTxs).ForEach(func(k, v []byte) error { - var txn util.Tx - err := txn.Deserialize(bytes.NewReader(v)) - if err != nil { - return err - } - txs = append(txs, &txn) - return nil - }) - }) - + it := t.db.NewIterator(dbutil.BytesPrefix(BKTTxs), nil) + defer it.Release() + for it.Next() { + var txn util.Tx + err := txn.Deserialize(bytes.NewReader(it.Value())) + if err != nil { + return nil, err + } + txs = append(txs, &txn) + } return txs, err } -func (t *txs) Del(txId *common.Uint256) (err error) { +func (t *txs) Del(txId *common.Uint256) error { t.Lock() defer t.Unlock() - return t.DB.Update(func(tx *bolt.Tx) error { - var txn util.Tx - data := tx.Bucket(BKTTxs).Get(txId.Bytes()) - err := txn.Deserialize(bytes.NewReader(data)) - if err != nil { - return err - } + var txn util.Tx + data, err := t.db.Get(toKey(BKTTxs, txId.Bytes()...), nil) + if err != nil { + return err + } - var key [4]byte - binary.BigEndian.PutUint32(key[:], txn.Height) - data = tx.Bucket(BKTHeightTxs).Get(key[:]) + if err := txn.Deserialize(bytes.NewReader(data)); err != nil { + return err + } - data = delTxId(data, &txn.Hash) + var key [4]byte + binary.BigEndian.PutUint32(key[:], txn.Height) + data, _ = t.db.Get(toKey(BKTHeightTxs, key[:]...), nil) - return tx.Bucket(BKTHeightTxs).Put(key[:], data) - }) + batch := new(leveldb.Batch) + batch.Delete(toKey(BKTTxs, txId.Bytes()...)) + batch.Put(toKey(BKTHeightTxs, key[:]...), delTxId(data, &txn.Hash)) + + return t.db.Write(batch, nil) } -func delTxId(data []byte, hash *common.Uint256) []byte{ +func delTxId(data []byte, hash *common.Uint256) []byte { // Get tx count var count uint16 if len(data) == 0 { @@ -184,7 +163,7 @@ func delTxId(data []byte, hash *common.Uint256) []byte{ data = data[2:] for i := uint16(0); i < count; i++ { var txId common.Uint256 - copy(txId[:],data[i*common.UINT256SIZE : (i+1)*common.UINT256SIZE]) + copy(txId[:], data[i*common.UINT256SIZE:(i+1)*common.UINT256SIZE]) if txId.IsEqual(*hash) { data = append(data[0:i*common.UINT256SIZE], data[(i+1)*common.UINT256SIZE:]...) break @@ -197,29 +176,27 @@ func delTxId(data []byte, hash *common.Uint256) []byte{ } func (t *txs) Batch() TxsBatch { - tx, err := t.DB.Begin(true) - if err != nil { - panic(err) - } - - return &txsBatch{Tx: tx} + return &txsBatch{DB: t.db, Batch: new(leveldb.Batch)} } func (t *txs) Clear() error { t.Lock() defer t.Unlock() - return t.DB.Update(func(tx *bolt.Tx) error { - if err := tx.DeleteBucket(BKTTxs); err != nil { - return err - } + it := t.db.NewIterator(dbutil.BytesPrefix(BKTTxs), nil) + batch := new(leveldb.Batch) + for it.Next() { + batch.Delete(it.Key()) + } + it.Release() - if err := tx.DeleteBucket(BKTHeightTxs); err != nil { - return err - } + it = t.db.NewIterator(dbutil.BytesPrefix(BKTHeightTxs), nil) + for it.Next() { + batch.Delete(it.Key()) + } + it.Release() - return nil - }) + return t.db.Write(batch, nil) } func (t *txs) Close() error { diff --git a/interface/store/txsbatch.go b/interface/store/txsbatch.go index 967c03a..50a0ca3 100644 --- a/interface/store/txsbatch.go +++ b/interface/store/txsbatch.go @@ -3,13 +3,12 @@ package store import ( "bytes" "encoding/binary" - "encoding/gob" - "github.com/boltdb/bolt" "sync" "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/syndtr/goleveldb/leveldb" ) // Ensure txsBatch implement TxsBatch interface. @@ -17,7 +16,8 @@ var _ TxsBatch = (*txsBatch)(nil) type txsBatch struct { sync.Mutex - *bolt.Tx + *leveldb.DB + *leveldb.Batch addTxs []*util.Tx delTxs []*util.Tx } @@ -31,11 +31,7 @@ func (b *txsBatch) Put(tx *util.Tx) error { return err } - err := b.Tx.Bucket(BKTTxs).Put(tx.Hash.Bytes(), buf.Bytes()) - if err != nil { - return err - } - + b.Batch.Put(toKey(BKTTxs, tx.Hash.Bytes()...), buf.Bytes()) b.addTxs = append(b.addTxs, tx) return nil } @@ -45,17 +41,15 @@ func (b *txsBatch) Del(txId *common.Uint256) error { defer b.Unlock() var tx util.Tx - data := b.Tx.Bucket(BKTTxs).Get(txId.Bytes()) - err := tx.Deserialize(bytes.NewReader(data)) + data, err := b.DB.Get(toKey(BKTTxs, txId.Bytes()...), nil) if err != nil { return err } - - err = b.Tx.Bucket(BKTTxs).Delete(txId.Bytes()) - if err != nil { + if err := tx.Deserialize(bytes.NewReader(data)); err != nil { return err } + b.Batch.Delete(toKey(BKTTxs, txId.Bytes()...)) b.delTxs = append(b.delTxs, &tx) return nil } @@ -66,52 +60,37 @@ func (b *txsBatch) DelAll(height uint32) error { var key [4]byte binary.BigEndian.PutUint32(key[:], height) - data := b.Tx.Bucket(BKTHeightTxs).Get(key[:]) - - var txMap = make(map[common.Uint256]uint32) - err := gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) - if err != nil { - return err + data, _ := b.DB.Get(toKey(BKTHeightTxs, key[:]...), nil) + for _, txID := range getTxIds(data) { + b.Batch.Delete(toKey(BKTTxs, txID.Bytes()...)) } + b.Batch.Delete(toKey(BKTHeightTxs, key[:]...)) - txBucket := b.Tx.Bucket(BKTTxs) - for txId := range txMap { - if err := txBucket.Delete(txId.Bytes()); err != nil { - return err - } - } + return nil +} - return b.Tx.Bucket(BKTHeightTxs).Delete(key[:]) +func (b *txsBatch) Rollback() error { + b.Lock() + defer b.Unlock() + b.Batch.Reset() + return nil } func (b *txsBatch) Commit() error { b.Lock() defer b.Unlock() - index := b.Tx.Bucket(BKTHeightTxs) - // Put height index for added transactions. if len(b.addTxs) > 0 { groups := groupByHeight(b.addTxs) for height, txs := range groups { var key [4]byte binary.BigEndian.PutUint32(key[:], height) - data := index.Get(key[:]) - - var txMap = make(map[common.Uint256]uint32) - // Ignore decode error, could be first adding. - gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) - + data, _ := b.DB.Get(toKey(BKTHeightTxs, key[:]...), nil) for _, tx := range txs { - txMap[tx.Hash] = height - } - - var buf = new(bytes.Buffer) - if err := gob.NewEncoder(buf).Encode(txMap); err != nil { - return err + data = putTxId(data, &tx.Hash) } - - return index.Put(key[:], buf.Bytes()) + b.Batch.Put(toKey(BKTHeightTxs, key[:]...), data) } } @@ -121,34 +100,15 @@ func (b *txsBatch) Commit() error { for height, txs := range groups { var key [4]byte binary.BigEndian.PutUint32(key[:], height) - data := index.Get(key[:]) - - var txMap = make(map[common.Uint256]uint32) - err := gob.NewDecoder(bytes.NewReader(data)).Decode(&txMap) - if err != nil { - return err - } - + data, _ := b.DB.Get(toKey(BKTHeightTxs, key[:]...), nil) for _, tx := range txs { - delete(txMap, tx.Hash) + data = delTxId(data, &tx.Hash) } - - var buf = new(bytes.Buffer) - if err = gob.NewEncoder(buf).Encode(txMap); err != nil { - return err - } - - return index.Put(key[:], buf.Bytes()) + b.Batch.Delete(toKey(BKTHeightTxs, key[:]...)) } } - return b.Tx.Commit() -} - -func (b *txsBatch) Rollback() error { - b.Lock() - defer b.Unlock() - return b.Tx.Rollback() + return b.DB.Write(b.Batch, nil) } func groupByHeight(txs []*util.Tx) map[uint32][]*util.Tx { diff --git a/wallet/store/headers/cache.go b/wallet/store/headers/cache.go index 6cafdb4..1160b7e 100644 --- a/wallet/store/headers/cache.go +++ b/wallet/store/headers/cache.go @@ -16,7 +16,7 @@ type cache struct { headers *ordered_map.OrderedMap } -func newHeaderCache(size int) *cache { +func newCache(size int) *cache { return &cache{ size: size, headers: ordered_map.NewOrderedMap(), diff --git a/wallet/store/headers/database.go b/wallet/store/headers/database.go index 80e9fd8..4f0317b 100644 --- a/wallet/store/headers/database.go +++ b/wallet/store/headers/database.go @@ -9,8 +9,8 @@ import ( "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/boltdb/bolt" "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/syndtr/goleveldb/leveldb" ) // Ensure Database implement headers interface @@ -19,40 +19,27 @@ var _ database.Headers = (*Database)(nil) // Headers implements Headers using bolt DB type Database struct { *sync.RWMutex - *bolt.DB - cache *cache - newBlockHeader func() util.BlockHeader + db *leveldb.DB + cache *cache + newHeader func() util.BlockHeader } var ( - BKTHeaders = []byte("Headers") - BKTChainTip = []byte("ChainTip") - KEYChainTip = []byte("ChainTip") + BKTHeaders = []byte("H") + BKTChainTip = []byte("B") ) -func NewDatabase(newBlockHeader func() util.BlockHeader) (*Database, error) { - db, err := bolt.Open("headers.bin", 0644, &bolt.Options{InitialMmapSize: 5000000}) +func NewDatabase(newHeader func() util.BlockHeader) (*Database, error) { + db, err := leveldb.OpenFile("HEADER", nil) if err != nil { return nil, err } - db.Update(func(btx *bolt.Tx) error { - _, err := btx.CreateBucketIfNotExists(BKTHeaders) - if err != nil { - return err - } - _, err = btx.CreateBucketIfNotExists(BKTChainTip) - if err != nil { - return err - } - return nil - }) - headers := &Database{ - RWMutex: new(sync.RWMutex), - DB: db, - cache: newHeaderCache(100), - newBlockHeader: newBlockHeader, + RWMutex: new(sync.RWMutex), + db: db, + cache: newCache(100), + newHeader: newHeader, } headers.initCache() @@ -87,27 +74,25 @@ func (d *Database) Put(header *util.Header, newTip bool) error { if newTip { d.cache.tip = header } - return d.Update(func(tx *bolt.Tx) error { + key := toKey(BKTHeaders, header.Hash().Bytes()...) - bytes, err := header.Serialize() - if err != nil { - return err - } + bytes, err := header.Serialize() + if err != nil { + return err + } + + err = d.db.Put(key, bytes, nil) + if err != nil { + return err + } - err = tx.Bucket(BKTHeaders).Put(header.Hash().Bytes(), bytes) + if newTip { + err = d.db.Put(BKTChainTip, bytes, nil) if err != nil { return err } - - if newTip { - err = tx.Bucket(BKTChainTip).Put(KEYChainTip, bytes) - if err != nil { - return err - } - } - - return nil - }) + } + return nil } func (d *Database) GetPrevious(header *util.Header) (*util.Header, error) { @@ -130,21 +115,7 @@ func (d *Database) Get(hash *common.Uint256) (header *util.Header, err error) { return header, nil } - err = d.View(func(tx *bolt.Tx) error { - - header, err = d.getHeader(tx, BKTHeaders, hash.Bytes()) - if err != nil { - return err - } - - return nil - }) - - if err != nil { - return nil, err - } - - return header, err + return d.getHeader(toKey(BKTHeaders, hash.Bytes()...)) } func (d *Database) GetBest() (header *util.Header, err error) { @@ -155,51 +126,46 @@ func (d *Database) GetBest() (header *util.Header, err error) { return d.cache.tip, nil } - err = d.View(func(tx *bolt.Tx) error { - header, err = d.getHeader(tx, BKTChainTip, KEYChainTip) - return err - }) - if err != nil { - return nil, fmt.Errorf("Headers db get tip error %s", err.Error()) - } - - return header, err + return d.getHeader(BKTChainTip) } func (d *Database) Clear() error { d.Lock() defer d.Unlock() - return d.Update(func(tx *bolt.Tx) error { - err := tx.DeleteBucket(BKTHeaders) - if err != nil { - return err - } - - return tx.DeleteBucket(BKTChainTip) - }) + batch := new(leveldb.Batch) + inter := d.db.NewIterator(nil, nil) + for inter.Next() { + batch.Delete(inter.Key()) + } + inter.Release() + return d.db.Write(batch, nil) } // Close db func (d *Database) Close() error { d.Lock() - err := d.DB.Close() - log.Debug("headers database closed") - return err + defer d.Unlock() + return d.db.Close() } -func (d *Database) getHeader(tx *bolt.Tx, bucket []byte, key []byte) (*util.Header, error) { - headerBytes := tx.Bucket(bucket).Get(key) - if headerBytes == nil { - return nil, fmt.Errorf("Header %s does not exist in database", hex.EncodeToString(key)) +func (d *Database) getHeader(key []byte) (*util.Header, error) { + data, err := d.db.Get(key, nil) + if err != nil { + return nil, fmt.Errorf("header %s does not exist in database", + hex.EncodeToString(key)) } var header util.Header - header.BlockHeader = d.newBlockHeader() - err := header.Deserialize(headerBytes) + header.BlockHeader = d.newHeader() + err = header.Deserialize(data) if err != nil { return nil, err } return &header, nil } + +func toKey(bucket []byte, index ...byte) []byte { + return append(bucket, index...) +} From 14448fd794acfc136fe29fa06fabcba4b2610cc5 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Fri, 7 Dec 2018 14:55:55 +0800 Subject: [PATCH 58/73] fix blockchain panic on first startup issue --- interface/store/headers.go | 7 ------- wallet/store/headers/database.go | 7 ------- 2 files changed, 14 deletions(-) diff --git a/interface/store/headers.go b/interface/store/headers.go index 168f6f2..7d7acb2 100644 --- a/interface/store/headers.go +++ b/interface/store/headers.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "errors" "fmt" - "math/big" "path/filepath" "sync" @@ -103,12 +102,6 @@ func (h *headers) Put(header *util.Header, newTip bool) error { } func (h *headers) GetPrevious(header *util.Header) (*util.Header, error) { - if header.Height == 0 { - return nil, fmt.Errorf("no more previous header") - } - if header.Height == 1 { - return &util.Header{TotalWork: new(big.Int)}, nil - } hash := header.Previous() return h.Get(&hash) } diff --git a/wallet/store/headers/database.go b/wallet/store/headers/database.go index 4f0317b..5d3aec6 100644 --- a/wallet/store/headers/database.go +++ b/wallet/store/headers/database.go @@ -3,7 +3,6 @@ package headers import ( "encoding/hex" "fmt" - "math/big" "sync" "github.com/elastos/Elastos.ELA.SPV/database" @@ -96,12 +95,6 @@ func (d *Database) Put(header *util.Header, newTip bool) error { } func (d *Database) GetPrevious(header *util.Header) (*util.Header, error) { - if header.Height == 0 { - return nil, fmt.Errorf("no more previous header") - } - if header.Height == 1 { - return &util.Header{TotalWork: new(big.Int)}, nil - } hash := header.Previous() return d.Get(&hash) } From 12857546b3174b2d6b4b783b996b1651d46d20e3 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Fri, 7 Dec 2018 16:32:35 +0800 Subject: [PATCH 59/73] decoupling Header and Transaction dependency from SDK --- bloom/filter.go | 108 ------------------------ bloom/merkleblock.go | 34 ++------ client.go | 6 +- database/chainstore.go | 2 +- interface/{blockheader.go => config.go} | 9 +- interface/iutil/header.go | 35 ++++++++ interface/iutil/tx.go | 51 +++++++++++ interface/spvservice.go | 25 +++--- sdk/service.go | 3 +- spvwallet.go | 40 ++++----- sync/config.go | 2 +- sync/manager.go | 3 +- util/elaheader.go | 33 -------- util/interface.go | 8 +- util/sideheader.go | 33 -------- wallet/client.go | 5 +- wallet/client/database/database.go | 7 +- wallet/client/wallet.go | 9 +- wallet/store/headers/database.go | 5 +- wallet/sutil/header.go | 39 +++++++++ wallet/sutil/tx.go | 51 +++++++++++ 21 files changed, 244 insertions(+), 264 deletions(-) rename interface/{blockheader.go => config.go} (91%) create mode 100644 interface/iutil/header.go create mode 100644 interface/iutil/tx.go delete mode 100644 util/elaheader.go delete mode 100644 util/sideheader.go create mode 100644 wallet/sutil/header.go create mode 100644 wallet/sutil/tx.go diff --git a/bloom/filter.go b/bloom/filter.go index c16f0b8..9dfc519 100644 --- a/bloom/filter.go +++ b/bloom/filter.go @@ -6,10 +6,8 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.SideChain/types" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA/core" ) const ( @@ -241,112 +239,6 @@ func (bf *Filter) AddOutPoint(outpoint *util.OutPoint) { bf.mtx.Unlock() } -// matchTxAndUpdate returns true if the bloom filter matches data within the -// passed tx, otherwise false is returned. If the filter does match -// the passed tx, it will also update the filter depending on the bloom -// update flags set via the loaded filter if needed. -// -// This function MUST be called with the filter lock held. -func (bf *Filter) matchElaTxAndUpdate(tx *core.Transaction) bool { - // Check if the filter matches the hash of the tx. - // This is useful for finding transactions when they appear in a block. - hash := tx.Hash() - matched := bf.matches(hash[:]) - - for i, txOut := range tx.Outputs { - if !bf.matches(txOut.ProgramHash[:]) { - continue - } - - matched = true - bf.addOutPoint(util.NewOutPoint(tx.Hash(), uint16(i))) - } - - // Nothing more to do if a match has already been made. - if matched { - return true - } - - // At this point, the tx and none of the data elements in the - // public key scripts of its outputs matched. - - // Check if the filter matches any outpoints this tx spends - for _, txIn := range tx.Inputs { - op := txIn.Previous - if bf.matchesOutPoint(util.NewOutPoint(op.TxID, op.Index)) { - return true - } - } - - return false -} - -// MatchTxAndUpdate returns true if the bloom filter matches data within the -// passed tx, otherwise false is returned. If the filter does match -// the passed tx, it will also update the filter depending on the bloom -// update flags set via the loaded filter if needed. -// -// This function is safe for concurrent access. -func (bf *Filter) MatchElaTxAndUpdate(tx *core.Transaction) bool { - bf.mtx.Lock() - match := bf.matchElaTxAndUpdate(tx) - bf.mtx.Unlock() - return match -} - -// matchTxAndUpdate returns true if the bloom filter matches data within the -// passed tx, otherwise false is returned. If the filter does match -// the passed tx, it will also update the filter depending on the bloom -// update flags set via the loaded filter if needed. -// -// This function MUST be called with the filter lock held. -func (bf *Filter) matchSideTxAndUpdate(tx *types.Transaction) bool { - // Check if the filter matches the hash of the tx. - // This is useful for finding transactions when they appear in a block. - hash := tx.Hash() - matched := bf.matches(hash[:]) - - for i, txOut := range tx.Outputs { - if !bf.matches(txOut.ProgramHash[:]) { - continue - } - - matched = true - bf.addOutPoint(util.NewOutPoint(tx.Hash(), uint16(i))) - } - - // Nothing more to do if a match has already been made. - if matched { - return true - } - - // At this point, the tx and none of the data elements in the - // public key scripts of its outputs matched. - - // Check if the filter matches any outpoints this tx spends - for _, txIn := range tx.Inputs { - op := txIn.Previous - if bf.matchesOutPoint(util.NewOutPoint(op.TxID, op.Index)) { - return true - } - } - - return false -} - -// MatchTxAndUpdate returns true if the bloom filter matches data within the -// passed tx, otherwise false is returned. If the filter does match -// the passed tx, it will also update the filter depending on the bloom -// update flags set via the loaded filter if needed. -// -// This function is safe for concurrent access. -func (bf *Filter) MatchSideTxAndUpdate(tx *types.Transaction) bool { - bf.mtx.Lock() - match := bf.matchSideTxAndUpdate(tx) - bf.mtx.Unlock() - return match -} - func (bf *Filter) GetFilterLoadMsg() *msg.FilterLoad { return bf.msg } diff --git a/bloom/merkleblock.go b/bloom/merkleblock.go index cde85b8..e8d662e 100644 --- a/bloom/merkleblock.go +++ b/bloom/merkleblock.go @@ -6,10 +6,8 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.SideChain/types" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA/core" ) // mBlock is used to house intermediate information needed to generate a @@ -102,31 +100,15 @@ func NewMerkleBlock(block *util.Block, filter *Filter) (*msg.MerkleBlock, []uint // Find and keep track of any transactions that match the filter. var matchedIndexes []uint32 - switch block.Header.BlockHeader.(type) { - case *util.ElaHeader: - for index, tx := range block.Transactions { - if filter.MatchElaTxAndUpdate(tx.(*core.Transaction)) { - mBlock.MatchedBits = append(mBlock.MatchedBits, 0x01) - matchedIndexes = append(matchedIndexes, uint32(index)) - } else { - mBlock.MatchedBits = append(mBlock.MatchedBits, 0x00) - } - txHash := tx.Hash() - mBlock.AllHashes = append(mBlock.AllHashes, &txHash) + for index, tx := range block.Transactions { + if tx.MatchFilter(filter) { + mBlock.MatchedBits = append(mBlock.MatchedBits, 0x01) + matchedIndexes = append(matchedIndexes, uint32(index)) + } else { + mBlock.MatchedBits = append(mBlock.MatchedBits, 0x00) } - - case *util.SideHeader: - for index, tx := range block.Transactions { - if filter.MatchSideTxAndUpdate(tx.(*types.Transaction)) { - mBlock.MatchedBits = append(mBlock.MatchedBits, 0x01) - matchedIndexes = append(matchedIndexes, uint32(index)) - } else { - mBlock.MatchedBits = append(mBlock.MatchedBits, 0x00) - } - txHash := tx.Hash() - mBlock.AllHashes = append(mBlock.AllHashes, &txHash) - } - + txHash := tx.Hash() + mBlock.AllHashes = append(mBlock.AllHashes, &txHash) } // Calculate the number of merkle branches (height) in the tree. diff --git a/client.go b/client.go index d077bec..103ad92 100644 --- a/client.go +++ b/client.go @@ -3,9 +3,7 @@ package main import ( "fmt" - "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.SPV/wallet" - "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA/core" ) @@ -14,9 +12,7 @@ var Version string func main() { url := fmt.Sprint("http://127.0.0.1:", config.JsonRpcPort, "/spvwallet") - wallet.RunClient(Version, url, getSystemAssetId(), func() util.BlockHeader { - return util.NewElaHeader(&core.Header{}) - }) + wallet.RunClient(Version, url, getSystemAssetId()) } func getSystemAssetId() common.Uint256 { diff --git a/database/chainstore.go b/database/chainstore.go index 6190272..a475774 100644 --- a/database/chainstore.go +++ b/database/chainstore.go @@ -18,7 +18,7 @@ type ChainStore interface { // Rollback delete all transactions after the reorg point, // it is used when blockchain reorganized. - Rollback(reorg *util.Header) error + Rollback(reorg *util.Header) error } func NewHeadersOnlyChainDB(db Headers) ChainStore { diff --git a/interface/blockheader.go b/interface/config.go similarity index 91% rename from interface/blockheader.go rename to interface/config.go index 5f72640..148036f 100644 --- a/interface/blockheader.go +++ b/interface/config.go @@ -3,6 +3,7 @@ package _interface import ( "time" + "github.com/elastos/Elastos.ELA.SPV/interface/iutil" "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.Utility/common" @@ -11,7 +12,11 @@ import ( ) func newBlockHeader() util.BlockHeader { - return util.NewElaHeader(&core.Header{}) + return iutil.NewHeader(&core.Header{}) +} + +func newTransaction() util.Transaction { + return iutil.NewTx(&core.Transaction{}) } // GenesisHeader creates a specific genesis header by the given @@ -87,5 +92,5 @@ func GenesisHeader(foundation *common.Uint168) util.BlockHeader { } header.MerkleRoot, _ = crypto.ComputeRoot(hashes) - return util.NewElaHeader(&header) + return iutil.NewHeader(&header) } diff --git a/interface/iutil/header.go b/interface/iutil/header.go new file mode 100644 index 0000000..a11114e --- /dev/null +++ b/interface/iutil/header.go @@ -0,0 +1,35 @@ +package iutil + +import ( + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" +) + +// Ensure Header implement BlockHeader interface. +var _ util.BlockHeader = (*Header)(nil) + +type Header struct { + *core.Header +} + +func (h *Header) Previous() common.Uint256 { + return h.Header.Previous +} + +func (h *Header) Bits() uint32 { + return h.Header.Bits +} + +func (h *Header) MerkleRoot() common.Uint256 { + return h.Header.MerkleRoot +} + +func (h *Header) PowHash() common.Uint256 { + return h.AuxPow.ParBlockHeader.Hash() +} + +func NewHeader(orgHeader *core.Header) util.BlockHeader { + return &Header{Header: orgHeader} +} diff --git a/interface/iutil/tx.go b/interface/iutil/tx.go new file mode 100644 index 0000000..668dace --- /dev/null +++ b/interface/iutil/tx.go @@ -0,0 +1,51 @@ +package iutil + +import ( + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/elastos/Elastos.ELA/core" +) + +var _ util.Transaction = (*Tx)(nil) + +type Tx struct { + *core.Transaction +} + +func (tx *Tx) MatchFilter(bf util.Filter) bool { + // Check if the filter matches the hash of the tx. + // This is useful for finding transactions when they appear in a block. + hash := tx.Hash() + matched := bf.Matches(hash[:]) + + for i, txOut := range tx.Outputs { + if !bf.Matches(txOut.ProgramHash[:]) { + continue + } + + matched = true + bf.Add(util.NewOutPoint(tx.Hash(), uint16(i)).Bytes()) + } + + // Nothing more to do if a match has already been made. + if matched { + return true + } + + // At this point, the tx and none of the data elements in the + // public key scripts of its outputs matched. + + // Check if the filter matches any outpoints this tx spends + for _, txIn := range tx.Inputs { + op := txIn.Previous + if bf.Matches(util.NewOutPoint(op.TxID, op.Index).Bytes()) { + return true + } + } + + return false +} + +func NewTx(tx *core.Transaction) *Tx { + return &Tx{tx} +} diff --git a/interface/spvservice.go b/interface/spvservice.go index f9ff449..d135b56 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -9,6 +9,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/bloom" "github.com/elastos/Elastos.ELA.SPV/database" + "github.com/elastos/Elastos.ELA.SPV/interface/iutil" "github.com/elastos/Elastos.ELA.SPV/interface/store" "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/util" @@ -70,16 +71,14 @@ func newSpvService(cfg *Config) (*spvservice, error) { chainStore := database.NewDefaultChainDB(headerStore, service) serviceCfg := &sdk.Config{ - DataDir: dataDir, - Magic: cfg.Magic, - SeedList: cfg.SeedList, - DefaultPort: cfg.DefaultPort, - MaxPeers: cfg.MaxConnections, - GenesisHeader: GenesisHeader(foundation), - ChainStore: chainStore, - NewTransaction: func() util.Transaction { - return &core.Transaction{} - }, + DataDir: dataDir, + Magic: cfg.Magic, + SeedList: cfg.SeedList, + DefaultPort: cfg.DefaultPort, + MaxPeers: cfg.MaxConnections, + GenesisHeader: GenesisHeader(foundation), + ChainStore: chainStore, + NewTransaction: newTransaction, NewBlockHeader: newBlockHeader, GetFilterData: service.GetFilterData, StateNotifier: service, @@ -149,7 +148,7 @@ func (s *spvservice) VerifyTransaction(proof bloom.MerkleProof, tx core.Transact } func (s *spvservice) SendTransaction(tx core.Transaction) error { - return s.IService.SendTransaction(&tx) + return s.IService.SendTransaction(iutil.NewTx(&tx)) } func (s *spvservice) GetTransaction(txId *common.Uint256) (*core.Transaction, error) { @@ -239,7 +238,7 @@ func (s *spvservice) BlockCommitted(block *util.Block) { log.Infof("Receive block %s height %d", block.Hash(), block.Height) for _, tx := range block.Transactions { for _, listener := range s.listeners { - s.queueMessageByListener(listener, tx.(*core.Transaction), block.Height) + s.queueMessageByListener(listener, tx.(*iutil.Tx).Transaction, block.Height) } } @@ -385,7 +384,7 @@ type txBatch struct { // PutTx add a store transaction operation into batch, and return // if it is a false positive and error. func (b *txBatch) PutTx(utx util.Transaction, height uint32) (bool, error) { - tx := utx.(*core.Transaction) + tx := utx.(*iutil.Tx) hits := make(map[common.Uint168]struct{}) ops := make(map[*util.OutPoint]common.Uint168) for index, output := range tx.Outputs { diff --git a/sdk/service.go b/sdk/service.go index 5b3cfc9..a3f8eef 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -2,6 +2,7 @@ package sdk import ( "fmt" + "math/rand" "os" "time" @@ -138,7 +139,7 @@ func (s *service) updateFilter() *bloom.Filter { addresses, outpoints := s.cfg.GetFilterData() elements := uint32(len(addresses) + len(outpoints)) - filter := bloom.NewFilter(elements, 0, 0) + filter := bloom.NewFilter(elements, rand.Uint32(), 0) for _, address := range addresses { filter.Add(address.Bytes()) } diff --git a/spvwallet.go b/spvwallet.go index 2127364..dc541f7 100644 --- a/spvwallet.go +++ b/spvwallet.go @@ -21,7 +21,7 @@ import ( ) const ( - MaxPeers = 12 + MaxPeers = 12 ) var ErrInvalidParameter = fmt.Errorf("invalide parameter") @@ -162,8 +162,8 @@ type txBatch struct { // PutTx add a store transaction operation into batch, and return // if it is a false positive and error. -func (b *txBatch) PutTx(mtx util.Transaction, height uint32) (bool, error) { - tx := mtx.(*core.Transaction) +func (b *txBatch) PutTx(utx util.Transaction, height uint32) (bool, error) { + tx := utx.(*sutil.Tx) txId := tx.Hash() hits := 0 @@ -272,18 +272,18 @@ func (w *spvwallet) sendTransaction(params httputil.Params) (interface{}, error) return nil, ErrInvalidParameter } - var tx core.Transaction + var tx = newTransaction() err = tx.Deserialize(bytes.NewReader(txBytes)) if err != nil { return nil, fmt.Errorf("deserialize transaction failed %s", err) } - return nil, w.SendTransaction(&tx) + return nil, w.SendTransaction(tx) } func NewWallet() (*spvwallet, error) { // Initialize headers db - headers, err := headers.NewDatabase(newBlockHeader) + headers, err := headers.NewDatabase() if err != nil { return nil, err } @@ -301,16 +301,16 @@ func NewWallet() (*spvwallet, error) { // Initialize spv service w.IService, err = sdk.NewService( &sdk.Config{ - Magic: config.Magic, - SeedList: config.SeedList, - DefaultPort: config.DefaultPort, - MaxPeers: MaxPeers, - GenesisHeader: GenesisHeader(), - ChainStore: chainStore, - NewTransaction: newTransaction, - NewBlockHeader: newBlockHeader, - GetFilterData: w.GetFilterData, - StateNotifier: &w, + Magic: config.Magic, + SeedList: config.SeedList, + DefaultPort: config.DefaultPort, + MaxPeers: MaxPeers, + GenesisHeader: GenesisHeader(), + ChainStore: chainStore, + NewTransaction: newTransaction, + NewBlockHeader: sutil.NewEmptyHeader, + GetFilterData: w.GetFilterData, + StateNotifier: &w, }) if err != nil { return nil, err @@ -327,12 +327,8 @@ func NewWallet() (*spvwallet, error) { return &w, nil } -func newBlockHeader() util.BlockHeader { - return util.NewElaHeader(&core.Header{}) -} - func newTransaction() util.Transaction { - return new(core.Transaction) + return sutil.NewTx(&core.Transaction{}) } // GenesisHeader creates a specific genesis header by the given @@ -408,5 +404,5 @@ func GenesisHeader() util.BlockHeader { } header.MerkleRoot, _ = crypto.ComputeRoot(hashes) - return util.NewElaHeader(&header) + return sutil.NewHeader(&header) } diff --git a/sync/config.go b/sync/config.go index c0dd495..048cc25 100644 --- a/sync/config.go +++ b/sync/config.go @@ -14,7 +14,7 @@ const ( type Config struct { Chain *blockchain.BlockChain - MaxPeers int + MaxPeers int UpdateFilter func() *bloom.Filter TransactionAnnounce func(tx util.Transaction) diff --git a/sync/manager.go b/sync/manager.go index ee143be..e59e6e8 100644 --- a/sync/manager.go +++ b/sync/manager.go @@ -345,6 +345,7 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { block := bmsg.block blockHash := block.Hash() if _, exists = state.requestedBlocks[blockHash]; !exists { + log.Warnf("Received unrequested block from peer %s", peer) peer.Disconnect() return } @@ -380,9 +381,7 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { if state.badBlockRate() > maxBadBlockRate { log.Warnf("Disconnecting from peer %s because he sent us too many bad blocks", peer) peer.Disconnect() - return } - log.Warnf("Received unrequested block from peer %s", peer) return } diff --git a/util/elaheader.go b/util/elaheader.go deleted file mode 100644 index 90c72c1..0000000 --- a/util/elaheader.go +++ /dev/null @@ -1,33 +0,0 @@ -package util - -import ( - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" -) - -// Ensure SideHeader implement BlockHeader interface. -var _ BlockHeader = (*ElaHeader)(nil) - -type ElaHeader struct { - *core.Header -} - -func (h *ElaHeader) Previous() common.Uint256 { - return h.Header.Previous -} - -func (h *ElaHeader) Bits() uint32 { - return h.Header.Bits -} - -func (h *ElaHeader) MerkleRoot() common.Uint256 { - return h.Header.MerkleRoot -} - -func (h *ElaHeader) PowHash() common.Uint256 { - return h.AuxPow.ParBlockHeader.Hash() -} - -func NewElaHeader(orgHeader *core.Header) BlockHeader { - return &ElaHeader{Header: orgHeader} -} diff --git a/util/interface.go b/util/interface.go index f8b9790..3eb4a38 100644 --- a/util/interface.go +++ b/util/interface.go @@ -16,8 +16,14 @@ type BlockHeader interface { Deserialize(r io.Reader) error } +type Filter interface { + Add(data []byte) + Matches(data []byte) bool +} + type Transaction interface { Hash() common.Uint256 Serialize(w io.Writer) error Deserialize(r io.Reader) error -} \ No newline at end of file + MatchFilter(filter Filter) bool +} diff --git a/util/sideheader.go b/util/sideheader.go deleted file mode 100644 index cbf28a8..0000000 --- a/util/sideheader.go +++ /dev/null @@ -1,33 +0,0 @@ -package util - -import ( - "github.com/elastos/Elastos.ELA.SideChain/types" - "github.com/elastos/Elastos.ELA.Utility/common" -) - -// Ensure SideHeader implement BlockHeader interface. -var _ BlockHeader = (*SideHeader)(nil) - -type SideHeader struct { - *types.Header -} - -func (h *SideHeader) Previous() common.Uint256 { - return h.Header.Previous -} - -func (h *SideHeader) Bits() uint32 { - return h.Header.Bits -} - -func (h *SideHeader) MerkleRoot() common.Uint256 { - return h.Header.MerkleRoot -} - -func (h *SideHeader) PowHash() common.Uint256 { - return h.SideAuxPow.MainBlockHeader.Hash() -} - -func NewSideHeader(orgHeader *types.Header) BlockHeader { - return &SideHeader{Header: orgHeader} -} diff --git a/wallet/client.go b/wallet/client.go index 5556eb4..1ea459f 100644 --- a/wallet/client.go +++ b/wallet/client.go @@ -1,7 +1,6 @@ package wallet import ( - "github.com/elastos/Elastos.ELA.SPV/util" "os" "github.com/elastos/Elastos.ELA.SPV/wallet/client" @@ -13,8 +12,8 @@ import ( "github.com/urfave/cli" ) -func RunClient(version, rpcUrl string, assetId common.Uint256, newBlockHeader func() util.BlockHeader) { - client.Setup(rpcUrl, assetId, newBlockHeader) +func RunClient(version, rpcUrl string, assetId common.Uint256) { + client.Setup(rpcUrl, assetId) app := cli.NewApp() app.Name = "ELASTOS SPV WALLET" diff --git a/wallet/client/database/database.go b/wallet/client/database/database.go index 47090a1..ea7e510 100644 --- a/wallet/client/database/database.go +++ b/wallet/client/database/database.go @@ -3,7 +3,6 @@ package database import ( "sync" - "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.SPV/wallet/store/headers" "github.com/elastos/Elastos.ELA.SPV/wallet/store/sqlite" "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" @@ -11,7 +10,7 @@ import ( "github.com/elastos/Elastos.ELA.Utility/common" ) -func New(newBlockHeader func() util.BlockHeader) (*database, error) { +func New() (*database, error) { dataStore, err := sqlite.NewDatabase() if err != nil { return nil, err @@ -20,14 +19,12 @@ func New(newBlockHeader func() util.BlockHeader) (*database, error) { return &database{ lock: new(sync.RWMutex), store: dataStore, - newBlockHeader: newBlockHeader, }, nil } type database struct { lock *sync.RWMutex store sqlite.DataStore - newBlockHeader func() util.BlockHeader } func (d *database) AddAddress(address *common.Uint168, script []byte, addrType int) error { @@ -83,7 +80,7 @@ func (d *database) Clear() error { d.lock.Lock() defer d.lock.Unlock() - headers, err := headers.NewDatabase(d.newBlockHeader) + headers, err := headers.NewDatabase() if err != nil { return err } diff --git a/wallet/client/wallet.go b/wallet/client/wallet.go index 94f61af..3f226f4 100644 --- a/wallet/client/wallet.go +++ b/wallet/client/wallet.go @@ -9,7 +9,6 @@ import ( "strconv" "github.com/elastos/Elastos.ELA.SPV/sdk" - "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.SPV/wallet/client/database" "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" @@ -22,7 +21,6 @@ import ( var ( jsonRpcUrl string sysAssetId common.Uint256 - newHeader func() util.BlockHeader ) type Transfer struct { @@ -35,10 +33,9 @@ type Wallet struct { *Keystore } -func Setup(rpcUrl string, assetId common.Uint256, newBlockHeader func() util.BlockHeader) { +func Setup(rpcUrl string, assetId common.Uint256) { jsonRpcUrl = rpcUrl sysAssetId = assetId - newHeader = newBlockHeader } func Create(password []byte) error { @@ -47,7 +44,7 @@ func Create(password []byte) error { return err } - db, err := database.New(newHeader) + db, err := database.New() if err != nil { return err } @@ -58,7 +55,7 @@ func Create(password []byte) error { } func Open() (*Wallet, error) { - db, err := database.New(newHeader) + db, err := database.New() if err != nil { return nil, err } diff --git a/wallet/store/headers/database.go b/wallet/store/headers/database.go index 5d3aec6..e21310b 100644 --- a/wallet/store/headers/database.go +++ b/wallet/store/headers/database.go @@ -7,6 +7,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" "github.com/elastos/Elastos.ELA.Utility/common" "github.com/syndtr/goleveldb/leveldb" @@ -28,7 +29,7 @@ var ( BKTChainTip = []byte("B") ) -func NewDatabase(newHeader func() util.BlockHeader) (*Database, error) { +func NewDatabase() (*Database, error) { db, err := leveldb.OpenFile("HEADER", nil) if err != nil { return nil, err @@ -38,7 +39,7 @@ func NewDatabase(newHeader func() util.BlockHeader) (*Database, error) { RWMutex: new(sync.RWMutex), db: db, cache: newCache(100), - newHeader: newHeader, + newHeader: sutil.NewEmptyHeader, } headers.initCache() diff --git a/wallet/sutil/header.go b/wallet/sutil/header.go new file mode 100644 index 0000000..105447d --- /dev/null +++ b/wallet/sutil/header.go @@ -0,0 +1,39 @@ +package sutil + +import ( + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/core" +) + +// Ensure Header implement BlockHeader interface. +var _ util.BlockHeader = (*Header)(nil) + +type Header struct { + *core.Header +} + +func (h *Header) Previous() common.Uint256 { + return h.Header.Previous +} + +func (h *Header) Bits() uint32 { + return h.Header.Bits +} + +func (h *Header) MerkleRoot() common.Uint256 { + return h.Header.MerkleRoot +} + +func (h *Header) PowHash() common.Uint256 { + return h.AuxPow.ParBlockHeader.Hash() +} + +func NewHeader(orgHeader *core.Header) util.BlockHeader { + return &Header{Header: orgHeader} +} + +func NewEmptyHeader() util.BlockHeader { + return &Header{&core.Header{}} +} diff --git a/wallet/sutil/tx.go b/wallet/sutil/tx.go new file mode 100644 index 0000000..f699986 --- /dev/null +++ b/wallet/sutil/tx.go @@ -0,0 +1,51 @@ +package sutil + +import ( + "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/elastos/Elastos.ELA/core" +) + +var _ util.Transaction = (*Tx)(nil) + +type Tx struct { + *core.Transaction +} + +func (tx *Tx) MatchFilter(bf util.Filter) bool { + // Check if the filter matches the hash of the tx. + // This is useful for finding transactions when they appear in a block. + hash := tx.Hash() + matched := bf.Matches(hash[:]) + + for i, txOut := range tx.Outputs { + if !bf.Matches(txOut.ProgramHash[:]) { + continue + } + + matched = true + bf.Add(util.NewOutPoint(tx.Hash(), uint16(i)).Bytes()) + } + + // Nothing more to do if a match has already been made. + if matched { + return true + } + + // At this point, the tx and none of the data elements in the + // public key scripts of its outputs matched. + + // Check if the filter matches any outpoints this tx spends + for _, txIn := range tx.Inputs { + op := txIn.Previous + if bf.Matches(util.NewOutPoint(op.TxID, op.Index).Bytes()) { + return true + } + } + + return false +} + +func NewTx(tx *core.Transaction) *Tx { + return &Tx{tx} +} From be93cb23e428428b355c70771f87f85db8def563 Mon Sep 17 00:00:00 2001 From: jiangzehua Date: Wed, 12 Dec 2018 15:18:05 +0800 Subject: [PATCH 60/73] fix listener notify error --- interface/spvservice.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/interface/spvservice.go b/interface/spvservice.go index d135b56..cd378c2 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -188,9 +188,10 @@ func (s *spvservice) GetFilterData() ([]*common.Uint168, []*util.OutPoint) { // of transactions within a block. func (s *spvservice) Batch() database.TxBatch { return &txBatch{ - db: s.db, - batch: s.db.Batch(), - rollback: s.rollback, + db: s.db, + batch: s.db.Batch(), + rollback: s.rollback, + listeners: s.listeners, } } @@ -235,13 +236,6 @@ func (s *spvservice) TransactionConfirmed(tx *util.Tx) {} // BlockCommitted will be invoked when a block and transactions within it are // successfully committed into database. func (s *spvservice) BlockCommitted(block *util.Block) { - log.Infof("Receive block %s height %d", block.Hash(), block.Height) - for _, tx := range block.Transactions { - for _, listener := range s.listeners { - s.queueMessageByListener(listener, tx.(*iutil.Tx).Transaction, block.Height) - } - } - // Look up for queued transactions items, err := s.db.Que().GetAll() if err != nil { From 762cb2d3ef5a9745e5e7a803668b13cb7389409a Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 13 Dec 2018 17:45:18 +0800 Subject: [PATCH 61/73] replace sqlite database with levelDB --- glide.yaml | 1 - interface/store/databatch.go | 18 +--- interface/store/datastore.go | 29 +----- interface/store/ops.go | 4 +- interface/store/que.go | 121 +++++++++++------------ interface/store/que_test.go | 179 +++++++++++++++++++++++++++++++++++ interface/store/quebatch.go | 53 +++++++++-- interface/store/txs.go | 4 +- 8 files changed, 287 insertions(+), 122 deletions(-) create mode 100644 interface/store/que_test.go diff --git a/glide.yaml b/glide.yaml index a89f515..123faf3 100644 --- a/glide.yaml +++ b/glide.yaml @@ -14,5 +14,4 @@ import: subpackages: - bloom - core -- package: github.com/mattn/go-sqlite3 - package: github.com/urfave/cli diff --git a/interface/store/databatch.go b/interface/store/databatch.go index 1db6baf..4b0fda9 100644 --- a/interface/store/databatch.go +++ b/interface/store/databatch.go @@ -2,7 +2,6 @@ package store import ( "bytes" - "database/sql" "encoding/binary" "sync" @@ -19,7 +18,6 @@ type dataBatch struct { mutex sync.Mutex *leveldb.DB *leveldb.Batch - sqlTx *sql.Tx } func (b *dataBatch) Txs() TxsBatch { @@ -31,7 +29,7 @@ func (b *dataBatch) Ops() OpsBatch { } func (b *dataBatch) Que() QueBatch { - return &queBatch{Tx: b.sqlTx} + return &queBatch{Batch: b.Batch} } // Delete all transactions, ops, queued items on the given height. @@ -72,20 +70,10 @@ func (b *dataBatch) DelAll(height uint32) error { } func (b *dataBatch) Commit() error { - defer b.sqlTx.Rollback() - - if err := b.DB.Write(b.Batch, nil); err != nil { - return err - } - - if err := b.sqlTx.Commit(); err != nil { - return err - } - - return nil + return b.DB.Write(b.Batch, nil) } func (b *dataBatch) Rollback() error { b.Batch.Reset() - return b.sqlTx.Rollback() + return nil } diff --git a/interface/store/datastore.go b/interface/store/datastore.go index 5ef2503..8e46a13 100644 --- a/interface/store/datastore.go +++ b/interface/store/datastore.go @@ -33,27 +33,12 @@ func NewDataStore(dataDir string) (*dataStore, error) { return nil, err } - txs, err := NewTxs(db) - if err != nil { - return nil, err - } - - ops, err := NewOps(db) - if err != nil { - return nil, err - } - - que, err := NewQue(dataDir) - if err != nil { - return nil, err - } - return &dataStore{ db: db, addrs: addrs, - txs: txs, - ops: ops, - que: que, + txs: NewTxs(db), + ops: NewOps(db), + que: NewQue(db), }, nil } @@ -74,15 +59,9 @@ func (d *dataStore) Que() Que { } func (d *dataStore) Batch() DataBatch { - sqlTx, err := d.que.Begin() - if err != nil { - panic(err) - } - return &dataBatch{ - DB:d.db, + DB: d.db, Batch: new(leveldb.Batch), - sqlTx: sqlTx, } } diff --git a/interface/store/ops.go b/interface/store/ops.go index 93cbcb1..0103beb 100644 --- a/interface/store/ops.go +++ b/interface/store/ops.go @@ -22,8 +22,8 @@ type ops struct { db *leveldb.DB } -func NewOps(db *leveldb.DB) (*ops, error) { - return &ops{db: db}, nil +func NewOps(db *leveldb.DB) *ops { + return &ops{db: db} } func (o *ops) Put(op *util.OutPoint, addr common.Uint168) error { diff --git a/interface/store/que.go b/interface/store/que.go index d0f6da6..ee98227 100644 --- a/interface/store/que.go +++ b/interface/store/que.go @@ -1,25 +1,19 @@ package store import ( - "database/sql" - "fmt" - "path/filepath" + "encoding/binary" "sync" "github.com/elastos/Elastos.ELA.Utility/common" - _ "github.com/mattn/go-sqlite3" -) -const ( - DriverName = "sqlite3" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" +) - CreateQueueDB = `CREATE TABLE IF NOT EXISTS Queue( - NotifyId BLOB NOT NULL, - TxId BLOB NOT NULL, - Height INTEGER NOT NULL); - CREATE INDEX IF NOT EXISTS idx_queue_notify_id ON Queue (NotifyId); - CREATE INDEX IF NOT EXISTS idx_queue_tx_id ON Queue (TxId); - CREATE INDEX IF NOT EXISTS idx_queue_height ON Queue (height);` +var ( + BKTQue = []byte("Q") + BKTQueIdx = []byte("U") + empty = make([]byte, 0) ) // Ensure que implement Que interface. @@ -27,21 +21,11 @@ var _ Que = (*que)(nil) type que struct { sync.RWMutex - *sql.DB + db *leveldb.DB } -func NewQue(dataDir string) (*que, error) { - db, err := sql.Open(DriverName, filepath.Join(dataDir, "queue.db")) - if err != nil { - fmt.Println("Open queue db error:", err) - return nil, err - } - - _, err = db.Exec(CreateQueueDB) - if err != nil { - return nil, err - } - return &que{DB: db}, nil +func NewQue(db *leveldb.DB) *que { + return &que{db: db} } // Put a queue item to database @@ -49,43 +33,30 @@ func (q *que) Put(item *QueItem) error { q.Lock() defer q.Unlock() - sql := "INSERT OR REPLACE INTO Queue(NotifyId, TxId, Height) VALUES(?,?,?)" - _, err := q.Exec(sql, item.NotifyId.Bytes(), item.TxId.Bytes(), item.Height) - return err + batch := new(leveldb.Batch) + var height [4]byte + binary.BigEndian.PutUint32(height[:], item.Height) + value := append(item.NotifyId[:], item.TxId[:]...) + batch.Put(toKey(BKTQue, value...), height[:]) + batch.Put(toKey(BKTQueIdx, append(height[:], value...)...), empty) + return q.db.Write(batch, nil) } // Get all items in queue -func (q *que) GetAll() ([]*QueItem, error) { +func (q *que) GetAll() (items []*QueItem, err error) { q.RLock() defer q.RUnlock() - rows, err := q.Query("SELECT NotifyId, TxId, Height FROM Queue") - if err != nil { - return nil, err - } - - var items []*QueItem - for rows.Next() { - var notifyIdBytes []byte - var txHashBytes []byte - var height uint32 - err = rows.Scan(¬ifyIdBytes, &txHashBytes, &height) - if err != nil { - return nil, err - } - - notifyId, err := common.Uint256FromBytes(notifyIdBytes) - if err != nil { - return nil, err - } - txHash, err := common.Uint256FromBytes(txHashBytes) - if err != nil { - return nil, err - } - item := &QueItem{NotifyId: *notifyId, TxId: *txHash, Height: height} - items = append(items, item) + it := q.db.NewIterator(util.BytesPrefix(BKTQue), nil) + defer it.Release() + for it.Next() { + var item QueItem + value := subKey(BKTQue, it.Key()) + copy(item.NotifyId[:], value[:32]) + copy(item.TxId[:], value[32:]) + item.Height = binary.BigEndian.Uint32(it.Value()) + items = append(items, &item) } - return items, nil } @@ -94,28 +65,42 @@ func (q *que) Del(notifyId, txHash *common.Uint256) error { q.Lock() defer q.Unlock() - _, err := q.Exec("DELETE FROM Queue WHERE NotifyId=? AND TxId=?", notifyId.Bytes(), txHash.Bytes()) - return err + value := append(notifyId[:], txHash[:]...) + height, err := q.db.Get(toKey(BKTQue, value...), nil) + if err != nil { + return err + } + batch := new(leveldb.Batch) + batch.Delete(toKey(BKTQue, value...)) + batch.Delete(toKey(BKTQueIdx, append(height[:], value...)...)) + return q.db.Write(batch, nil) } func (q *que) Batch() QueBatch { - tx, err := q.Begin() - if err != nil { - panic(err) - } - return &queBatch{Tx: tx} + return &queBatch{DB: q.db, Batch: new(leveldb.Batch)} } func (q *que) Clear() error { q.Lock() defer q.Unlock() - _, err := q.Exec("DROP TABLE if EXISTS Queue") - return err + batch := new(leveldb.Batch) + it := q.db.NewIterator(util.BytesPrefix(BKTQue), nil) + for it.Next() { + batch.Delete(it.Key()) + } + it.Release() + + it = q.db.NewIterator(util.BytesPrefix(BKTQueIdx), nil) + for it.Next() { + batch.Delete(it.Key()) + } + it.Release() + + return q.db.Write(batch, nil) } func (q *que) Close() error { q.Lock() - defer q.Unlock() - return q.DB.Close() + return nil } diff --git a/interface/store/que_test.go b/interface/store/que_test.go new file mode 100644 index 0000000..51551ac --- /dev/null +++ b/interface/store/que_test.go @@ -0,0 +1,179 @@ +package store + +import ( + "crypto/rand" + "testing" + + "github.com/elastos/Elastos.ELA.Utility/common" + + "github.com/stretchr/testify/assert" + "github.com/syndtr/goleveldb/leveldb" +) + +func TestQue(t *testing.T) { + db, err := leveldb.OpenFile("test", nil) + if !assert.NoError(t, err) { + t.FailNow() + } + que := NewQue(db) + if !assert.NoError(t, err) { + t.FailNow() + } + + notifyIDs := make([]common.Uint256, 100) + txHashes := make([]common.Uint256, 100) + for i := range notifyIDs { + rand.Read(notifyIDs[i][:]) + } + for i := range txHashes { + rand.Read(txHashes[i][:]) + } + + idIndex := make(map[common.Uint256]common.Uint256) + for i, notifyID := range notifyIDs { + que.Put(&QueItem{NotifyId: notifyID, TxId: txHashes[i]}) + idIndex[notifyIDs[i]] = txHashes[i] + } + + items, err := que.GetAll() + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, 100, len(items)) { + t.FailNow() + } + for _, item := range items { + txID, ok := idIndex[item.NotifyId] + if !assert.Equal(t, true, ok) { + t.FailNow() + } + if !assert.Equal(t, item.TxId, txID) { + t.FailNow() + } + } + + height0 := make([]byte, 4) + for i, notifyID := range notifyIDs { + value := append(notifyID[:], txHashes[i][:]...) + height, err := que.db.Get(toKey(BKTQue, value...), nil) + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, height0, height) { + t.FailNow() + } + } + + for i, notifyID := range notifyIDs { + que.Del(¬ifyID, &txHashes[i]) + if i+1 >= 50 { + break + } + } + + for i, notifyID := range notifyIDs { + value := append(notifyID[:], txHashes[i][:]...) + _, err := que.db.Get(toKey(BKTQue, value...), nil) + if i < 50 { + if !assert.Error(t, err) { + t.FailNow() + } + continue + } + + if !assert.NoError(t, err) { + t.FailNow() + } + } + + items, err = que.GetAll() + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, 50, len(items)) { + t.FailNow() + } + + err = que.Clear() + if !assert.NoError(t, err) { + t.FailNow() + } + + items, err = que.GetAll() + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, 0, len(items)) { + t.FailNow() + } + + batch := que.Batch() + for i, notifyID := range notifyIDs { + batch.Put(&QueItem{NotifyId: notifyID, TxId: txHashes[i], Height: uint32(i)}) + } + err = batch.Commit() + if !assert.NoError(t, err) { + t.FailNow() + } + + items, err = que.GetAll() + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, 100, len(items)) { + t.FailNow() + } + for _, item := range items { + txID, ok := idIndex[item.NotifyId] + if !assert.Equal(t, true, ok) { + t.FailNow() + } + if !assert.Equal(t, item.TxId, txID) { + t.FailNow() + } + } + + batch = que.Batch() + for i, notifyID := range notifyIDs { + batch.Del(¬ifyID, &txHashes[i]) + if i+1 >= 50 { + break + } + } + err = batch.Commit() + if !assert.NoError(t, err) { + t.FailNow() + } + + for i, notifyID := range notifyIDs { + value := append(notifyID[:], txHashes[i][:]...) + _, err := que.db.Get(toKey(BKTQue, value...), nil) + if i < 50 { + if !assert.Error(t, err) { + t.FailNow() + } + continue + } + + if !assert.NoError(t, err) { + t.FailNow() + } + } + + batch = que.Batch() + for i := 50; i < 100; i++ { + batch.DelAll(uint32(i)) + } + err = batch.Commit() + if !assert.NoError(t, err) { + t.FailNow() + } + + items, err = que.GetAll() + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, 0, len(items)) { + t.FailNow() + } +} diff --git a/interface/store/quebatch.go b/interface/store/quebatch.go index f16c162..ab46009 100644 --- a/interface/store/quebatch.go +++ b/interface/store/quebatch.go @@ -1,10 +1,13 @@ package store import ( - "database/sql" + "encoding/binary" "sync" "github.com/elastos/Elastos.ELA.Utility/common" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" ) // Ensure queBatch implement QueBatch interface. @@ -12,7 +15,8 @@ var _ QueBatch = (*queBatch)(nil) type queBatch struct { sync.Mutex - *sql.Tx + *leveldb.DB + *leveldb.Batch } // Put a queue item to database @@ -20,9 +24,12 @@ func (b *queBatch) Put(item *QueItem) error { b.Lock() defer b.Unlock() - sql := "INSERT OR REPLACE INTO Queue(NotifyId, TxId, Height) VALUES(?,?,?)" - _, err := b.Tx.Exec(sql, item.NotifyId.Bytes(), item.TxId.Bytes(), item.Height) - return err + var height [4]byte + binary.BigEndian.PutUint32(height[:], item.Height) + value := append(item.NotifyId[:], item.TxId[:]...) + b.Batch.Put(toKey(BKTQue, value...), height[:]) + b.Batch.Put(toKey(BKTQueIdx, append(height[:], value...)...), empty) + return nil } // Delete confirmed item in queue @@ -30,8 +37,14 @@ func (b *queBatch) Del(notifyId, txHash *common.Uint256) error { b.Lock() defer b.Unlock() - _, err := b.Tx.Exec("DELETE FROM Queue WHERE NotifyId=? AND TxId=?", notifyId.Bytes(), txHash.Bytes()) - return err + value := append(notifyId[:], txHash[:]...) + height, err := b.DB.Get(toKey(BKTQue, value...), nil) + if err != nil { + return err + } + b.Batch.Delete(toKey(BKTQue, value...)) + b.Batch.Delete(toKey(BKTQueIdx, append(height[:], value...)...)) + return nil } // Delete all items on the given height. @@ -39,6 +52,28 @@ func (b *queBatch) DelAll(height uint32) error { b.Lock() defer b.Unlock() - _, err := b.Tx.Exec("DELETE FROM Queue WHERE Height=?", height) - return err + var key [4]byte + binary.BigEndian.PutUint32(key[:], height) + prefix := toKey(BKTQueIdx, key[:]...) + it := b.DB.NewIterator(util.BytesPrefix(prefix), nil) + for it.Next() { + value := subKey(prefix, it.Key()) + b.Batch.Delete(toKey(BKTQue, value...)) + b.Batch.Delete(it.Key()) + } + it.Release() + return nil +} + +func (b *queBatch) Rollback() error { + b.Lock() + defer b.Unlock() + b.Batch.Reset() + return nil +} + +func (b *queBatch) Commit() error { + b.Lock() + defer b.Unlock() + return b.Write(b.Batch, nil) } diff --git a/interface/store/txs.go b/interface/store/txs.go index ba8e497..86d7a5d 100644 --- a/interface/store/txs.go +++ b/interface/store/txs.go @@ -25,8 +25,8 @@ type txs struct { db *leveldb.DB } -func NewTxs(db *leveldb.DB) (*txs, error) { - return &txs{db: db}, nil +func NewTxs(db *leveldb.DB) *txs { + return &txs{db: db} } func (t *txs) Put(txn *util.Tx) error { From 4aa5a057a7a98f018ce504f4b0cf0e1f5671dbac Mon Sep 17 00:00:00 2001 From: AlexPan Date: Fri, 14 Dec 2018 20:38:45 +0800 Subject: [PATCH 62/73] do not resend notify until receipt timeout --- interface/interface_test.go | 10 +++---- interface/spvservice.go | 54 +++++++++++++++++++++++------------- interface/store/interface.go | 9 ++++-- interface/store/que.go | 18 ++++++++---- interface/store/que_test.go | 28 +++++++++++++++---- 5 files changed, 81 insertions(+), 38 deletions(-) diff --git a/interface/interface_test.go b/interface/interface_test.go index 13b0063..714cc72 100644 --- a/interface/interface_test.go +++ b/interface/interface_test.go @@ -1,7 +1,6 @@ package _interface import ( - "fmt" "math/rand" "os" "testing" @@ -16,7 +15,6 @@ import ( "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/elalog" - "github.com/elastos/Elastos.ELA.Utility/http/jsonrpc" "github.com/elastos/Elastos.ELA.Utility/p2p/addrmgr" "github.com/elastos/Elastos.ELA.Utility/p2p/connmgr" "github.com/elastos/Elastos.ELA.Utility/p2p/server" @@ -27,6 +25,7 @@ import ( type TxListener struct { t *testing.T + log elalog.Logger service SPVService address string txType core.TransactionType @@ -46,7 +45,7 @@ func (l *TxListener) Flags() uint64 { } func (l *TxListener) Notify(id common.Uint256, proof bloom.MerkleProof, tx core.Transaction) { - fmt.Printf("Receive notify ID: %s, Type: %s\n", id.String(), tx.TxType.Name()) + l.log.Infof("Notify Type %s, TxID %s", tx.TxType.Name(), tx.Hash()) err := l.service.VerifyTransaction(proof, tx) if !assert.NoError(l.t, err) { l.t.FailNow() @@ -135,13 +134,12 @@ func TestNewSPVService(t *testing.T) { peerlog := backend.Logger("PEER", elalog.LevelDebug) spvslog := backend.Logger("SPVS", elalog.LevelDebug) srvrlog := backend.Logger("SRVR", elalog.LevelDebug) - rpcslog := backend.Logger("RPCS", elalog.LevelDebug) + listlog := backend.Logger("RPCS", elalog.LevelDebug) addrmgr.UseLogger(admrlog) connmgr.UseLogger(cmgrlog) blockchain.UseLogger(bcdblog) sdk.UseLogger(spvslog) - jsonrpc.UseLogger(rpcslog) peer.UseLogger(peerlog) server.UseLogger(srvrlog) store.UseLogger(bcdblog) @@ -173,6 +171,7 @@ func TestNewSPVService(t *testing.T) { confirmedListener := &TxListener{ t: t, + log: listlog, service: service, address: "8ZNizBf4KhhPjeJRGpox6rPcHE5Np6tFx3", txType: core.CoinBase, @@ -181,6 +180,7 @@ func TestNewSPVService(t *testing.T) { unconfirmedListener := &TxListener{ t: t, + log: listlog, service: service, address: "8ZNizBf4KhhPjeJRGpox6rPcHE5Np6tFx3", txType: core.TransferAsset, diff --git a/interface/spvservice.go b/interface/spvservice.go index cd378c2..c5bc530 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "os" + "time" "github.com/elastos/Elastos.ELA.SPV/bloom" "github.com/elastos/Elastos.ELA.SPV/database" @@ -19,7 +20,13 @@ import ( "github.com/elastos/Elastos.ELA/core" ) -const defaultDataDir = "./data_spv" +const ( + defaultDataDir = "./data_spv" + + // notifyTimeout is the duration to timeout a notify to the listener, and + // resend the notify to the listener. + notifyTimeout = 10 * time.Second // 10 second +) type spvservice struct { sdk.IService @@ -242,6 +249,11 @@ func (s *spvservice) BlockCommitted(block *util.Block) { return } for _, item := range items { + // Check if the notify should be resend due to timeout. + if time.Now().Before(item.LastNotify.Add(notifyTimeout)) { + continue + } + // Get header header, err := s.headers.GetByHeight(item.Height) if err != nil { @@ -262,19 +274,19 @@ func (s *spvservice) BlockCommitted(block *util.Block) { continue } + var proof = bloom.MerkleProof{ + BlockHash: header.Hash(), + Height: header.Height, + Transactions: header.NumTxs, + Hashes: header.Hashes, + Flags: header.Flags, + } + // Notify listeners - s.notifyTransaction( - item.NotifyId, - bloom.MerkleProof{ - BlockHash: header.Hash(), - Height: header.Height, - Transactions: header.NumTxs, - Hashes: header.Hashes, - Flags: header.Flags, - }, - tx, - block.Height-item.Height, - ) + if s.notifyTransaction(item.NotifyId, proof, tx, block.Height-item.Height) { + item.LastNotify = time.Now() + s.db.Que().Put(item) + } } } @@ -316,12 +328,12 @@ func (s *spvservice) queueMessageByListener( }) } -func (s *spvservice) notifyTransaction( - notifyId common.Uint256, proof bloom.MerkleProof, tx core.Transaction, confirmations uint32) { +func (s *spvservice) notifyTransaction(notifyId common.Uint256, + proof bloom.MerkleProof, tx core.Transaction, confirmations uint32) bool { listener, ok := s.listeners[notifyId] if !ok { - return + return false } // Get transaction id @@ -338,17 +350,21 @@ func (s *spvservice) notifyTransaction( } else { s.db.Que().Del(¬ifyId, &txId) } - return + return false } // Notify listener if listener.Flags()&FlagNotifyConfirmed == FlagNotifyConfirmed { if confirmations >= getConfirmations(tx) { - go listener.Notify(notifyId, proof, tx) + listener.Notify(notifyId, proof, tx) + return true } } else { - go listener.Notify(notifyId, proof, tx) + listener.Notify(notifyId, proof, tx) + return true } + + return false } func getListenerKey(listener TransactionListener) common.Uint256 { diff --git a/interface/store/interface.go b/interface/store/interface.go index 7e17bb3..48c2b35 100644 --- a/interface/store/interface.go +++ b/interface/store/interface.go @@ -1,6 +1,8 @@ package store import ( + "time" + "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/util" @@ -105,7 +107,8 @@ type QueBatch interface { } type QueItem struct { - NotifyId common.Uint256 - TxId common.Uint256 - Height uint32 + NotifyId common.Uint256 + TxId common.Uint256 + Height uint32 + LastNotify time.Time } diff --git a/interface/store/que.go b/interface/store/que.go index ee98227..a898cf0 100644 --- a/interface/store/que.go +++ b/interface/store/que.go @@ -1,11 +1,12 @@ package store import ( + "bytes" "encoding/binary" "sync" + "time" "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) @@ -34,11 +35,12 @@ func (q *que) Put(item *QueItem) error { defer q.Unlock() batch := new(leveldb.Batch) - var height [4]byte - binary.BigEndian.PutUint32(height[:], item.Height) + buf := new(bytes.Buffer) + binary.Write(buf, binary.BigEndian, item.Height) value := append(item.NotifyId[:], item.TxId[:]...) - batch.Put(toKey(BKTQue, value...), height[:]) - batch.Put(toKey(BKTQueIdx, append(height[:], value...)...), empty) + batch.Put(toKey(BKTQueIdx, append(buf.Bytes(), value...)...), empty) + binary.Write(buf, binary.BigEndian, item.LastNotify.Unix()) + batch.Put(toKey(BKTQue, value...), buf.Bytes()) return q.db.Write(batch, nil) } @@ -51,10 +53,14 @@ func (q *que) GetAll() (items []*QueItem, err error) { defer it.Release() for it.Next() { var item QueItem + var lastNotify int64 value := subKey(BKTQue, it.Key()) copy(item.NotifyId[:], value[:32]) copy(item.TxId[:], value[32:]) - item.Height = binary.BigEndian.Uint32(it.Value()) + buf := bytes.NewReader(it.Value()) + binary.Read(buf, binary.BigEndian, &item.Height) + binary.Read(buf, binary.BigEndian, &lastNotify) + item.LastNotify = time.Unix(lastNotify, 0) items = append(items, &item) } return items, nil diff --git a/interface/store/que_test.go b/interface/store/que_test.go index 51551ac..61962aa 100644 --- a/interface/store/que_test.go +++ b/interface/store/que_test.go @@ -2,7 +2,9 @@ package store import ( "crypto/rand" + "encoding/binary" "testing" + "time" "github.com/elastos/Elastos.ELA.Utility/common" @@ -19,6 +21,7 @@ func TestQue(t *testing.T) { if !assert.NoError(t, err) { t.FailNow() } + defer que.Clear() notifyIDs := make([]common.Uint256, 100) txHashes := make([]common.Uint256, 100) @@ -31,7 +34,11 @@ func TestQue(t *testing.T) { idIndex := make(map[common.Uint256]common.Uint256) for i, notifyID := range notifyIDs { - que.Put(&QueItem{NotifyId: notifyID, TxId: txHashes[i]}) + item := QueItem{NotifyId: notifyID, TxId: txHashes[i]} + if i%2 == 1 { + item.LastNotify = item.LastNotify.Add(time.Second) + } + que.Put(&item) idIndex[notifyIDs[i]] = txHashes[i] } @@ -52,16 +59,27 @@ func TestQue(t *testing.T) { } } - height0 := make([]byte, 4) + var defaultTime time.Time + data0 := make([]byte, 12) + data1 := make([]byte, 12) + binary.BigEndian.PutUint64(data0[4:], uint64(defaultTime.Unix())) + binary.BigEndian.PutUint64(data1[4:], uint64(defaultTime.Add(time.Second).Unix())) for i, notifyID := range notifyIDs { value := append(notifyID[:], txHashes[i][:]...) - height, err := que.db.Get(toKey(BKTQue, value...), nil) + data, err := que.db.Get(toKey(BKTQue, value...), nil) if !assert.NoError(t, err) { t.FailNow() } - if !assert.Equal(t, height0, height) { - t.FailNow() + if i%2 == 0 { + if !assert.Equal(t, data0, data) { + t.FailNow() + } + } else { + if !assert.Equal(t, data1, data) { + t.FailNow() + } } + } for i, notifyID := range notifyIDs { From 6d8f75479624a066622b6a19196f47aab7d7b5e2 Mon Sep 17 00:00:00 2001 From: jiangzehua Date: Mon, 17 Dec 2018 17:44:36 +0800 Subject: [PATCH 63/73] print spv logs --- interface/log.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/interface/log.go b/interface/log.go index de15bcd..7e24cd1 100644 --- a/interface/log.go +++ b/interface/log.go @@ -1,6 +1,12 @@ package _interface import ( + "github.com/elastos/Elastos.ELA.SPV/blockchain" + "github.com/elastos/Elastos.ELA.SPV/peer" + "github.com/elastos/Elastos.ELA.SPV/sdk" + "github.com/elastos/Elastos.ELA.SPV/sync" + "github.com/elastos/Elastos.ELA.SPV/wallet" + "github.com/elastos/Elastos.ELA.SPV/wallet/store" "github.com/elastos/Elastos.ELA.Utility/elalog" ) @@ -25,4 +31,10 @@ func DisableLog() { // using elalog. func UseLogger(logger elalog.Logger) { log = logger + blockchain.UseLogger(logger) + sdk.UseLogger(logger) + peer.UseLogger(logger) + store.UseLogger(logger) + sync.UseLogger(logger) + wallet.UseLogger(logger) } From 85df530c023045b2442b1eb85772a9d4350801e4 Mon Sep 17 00:00:00 2001 From: jiangzehua Date: Wed, 19 Dec 2018 12:04:55 +0800 Subject: [PATCH 64/73] rename log dir --- interface/store/datastore.go | 2 +- interface/store/headers.go | 2 +- wallet/store/headers/database.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/interface/store/datastore.go b/interface/store/datastore.go index 8e46a13..a28c62e 100644 --- a/interface/store/datastore.go +++ b/interface/store/datastore.go @@ -20,7 +20,7 @@ type dataStore struct { } func NewDataStore(dataDir string) (*dataStore, error) { - db, err := leveldb.OpenFile(filepath.Join(dataDir, "DATA"), nil) + db, err := leveldb.OpenFile(filepath.Join(dataDir, "store"), nil) if err != nil { return nil, err } diff --git a/interface/store/headers.go b/interface/store/headers.go index 7d7acb2..570597f 100644 --- a/interface/store/headers.go +++ b/interface/store/headers.go @@ -32,7 +32,7 @@ type headers struct { } func NewHeaderStore(dataDir string, newHeader func() util.BlockHeader) (*headers, error) { - db, err := leveldb.OpenFile(filepath.Join(dataDir, "HEADER"), nil) + db, err := leveldb.OpenFile(filepath.Join(dataDir, "header"), nil) if err != nil { return nil, err } diff --git a/wallet/store/headers/database.go b/wallet/store/headers/database.go index e21310b..77fbdae 100644 --- a/wallet/store/headers/database.go +++ b/wallet/store/headers/database.go @@ -30,7 +30,7 @@ var ( ) func NewDatabase() (*Database, error) { - db, err := leveldb.OpenFile("HEADER", nil) + db, err := leveldb.OpenFile("header", nil) if err != nil { return nil, err } From b1cf1bb70d65ad5aa8f36be3f49e5d2975764f7d Mon Sep 17 00:00:00 2001 From: AlexPan Date: Sat, 22 Dec 2018 16:33:53 +0800 Subject: [PATCH 65/73] fix que item not deleted after notify --- interface/spvservice.go | 20 +++++++++++--------- interface/store/que_test.go | 25 ++++++++++++++----------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/interface/spvservice.go b/interface/spvservice.go index c5bc530..623656b 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -25,7 +25,7 @@ const ( // notifyTimeout is the duration to timeout a notify to the listener, and // resend the notify to the listener. - notifyTimeout = 10 * time.Second // 10 second + notifyTimeout = 10 * time.Second // 10 second ) type spvservice struct { @@ -283,9 +283,11 @@ func (s *spvservice) BlockCommitted(block *util.Block) { } // Notify listeners - if s.notifyTransaction(item.NotifyId, proof, tx, block.Height-item.Height) { + listener, ok := s.notifyTransaction(item.NotifyId, proof, tx, block.Height-item.Height) + if ok { item.LastNotify = time.Now() s.db.Que().Put(item) + listener.Notify(item.NotifyId, proof, tx) } } } @@ -329,11 +331,12 @@ func (s *spvservice) queueMessageByListener( } func (s *spvservice) notifyTransaction(notifyId common.Uint256, - proof bloom.MerkleProof, tx core.Transaction, confirmations uint32) bool { + proof bloom.MerkleProof, tx core.Transaction, + confirmations uint32) (TransactionListener, bool) { listener, ok := s.listeners[notifyId] if !ok { - return false + return nil, false } // Get transaction id @@ -350,21 +353,20 @@ func (s *spvservice) notifyTransaction(notifyId common.Uint256, } else { s.db.Que().Del(¬ifyId, &txId) } - return false + return nil, false } // Notify listener if listener.Flags()&FlagNotifyConfirmed == FlagNotifyConfirmed { if confirmations >= getConfirmations(tx) { - listener.Notify(notifyId, proof, tx) - return true + return listener, true } } else { listener.Notify(notifyId, proof, tx) - return true + return listener, true } - return false + return nil, false } func getListenerKey(listener TransactionListener) common.Uint256 { diff --git a/interface/store/que_test.go b/interface/store/que_test.go index 61962aa..bcfa440 100644 --- a/interface/store/que_test.go +++ b/interface/store/que_test.go @@ -23,8 +23,10 @@ func TestQue(t *testing.T) { } defer que.Clear() - notifyIDs := make([]common.Uint256, 100) - txHashes := make([]common.Uint256, 100) + times := 1000 + + notifyIDs := make([]common.Uint256, times) + txHashes := make([]common.Uint256, times) for i := range notifyIDs { rand.Read(notifyIDs[i][:]) } @@ -46,7 +48,7 @@ func TestQue(t *testing.T) { if !assert.NoError(t, err) { t.FailNow() } - if !assert.Equal(t, 100, len(items)) { + if !assert.Equal(t, times, len(items)) { t.FailNow() } for _, item := range items { @@ -83,8 +85,9 @@ func TestQue(t *testing.T) { } for i, notifyID := range notifyIDs { - que.Del(¬ifyID, &txHashes[i]) - if i+1 >= 50 { + err := que.Del(¬ifyID, &txHashes[i]) + assert.NoError(t, err) + if i+1 >= times/2 { break } } @@ -92,7 +95,7 @@ func TestQue(t *testing.T) { for i, notifyID := range notifyIDs { value := append(notifyID[:], txHashes[i][:]...) _, err := que.db.Get(toKey(BKTQue, value...), nil) - if i < 50 { + if i < times/2 { if !assert.Error(t, err) { t.FailNow() } @@ -108,7 +111,7 @@ func TestQue(t *testing.T) { if !assert.NoError(t, err) { t.FailNow() } - if !assert.Equal(t, 50, len(items)) { + if !assert.Equal(t, times/2, len(items)) { t.FailNow() } @@ -138,7 +141,7 @@ func TestQue(t *testing.T) { if !assert.NoError(t, err) { t.FailNow() } - if !assert.Equal(t, 100, len(items)) { + if !assert.Equal(t, times, len(items)) { t.FailNow() } for _, item := range items { @@ -154,7 +157,7 @@ func TestQue(t *testing.T) { batch = que.Batch() for i, notifyID := range notifyIDs { batch.Del(¬ifyID, &txHashes[i]) - if i+1 >= 50 { + if i+1 >= times/2 { break } } @@ -166,7 +169,7 @@ func TestQue(t *testing.T) { for i, notifyID := range notifyIDs { value := append(notifyID[:], txHashes[i][:]...) _, err := que.db.Get(toKey(BKTQue, value...), nil) - if i < 50 { + if i < times/2 { if !assert.Error(t, err) { t.FailNow() } @@ -179,7 +182,7 @@ func TestQue(t *testing.T) { } batch = que.Batch() - for i := 50; i < 100; i++ { + for i := times / 2; i < times; i++ { batch.DelAll(uint32(i)) } err = batch.Commit() From db86f0325b9fddf7eef5bed5b4277de1a8097db3 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 25 Dec 2018 10:39:46 +0800 Subject: [PATCH 66/73] fix DataBatch gose panic when call DelAll() method issue #49 --- interface/store/databatch.go | 6 +++--- interface/store/databatch_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 interface/store/databatch_test.go diff --git a/interface/store/databatch.go b/interface/store/databatch.go index 4b0fda9..5786ae3 100644 --- a/interface/store/databatch.go +++ b/interface/store/databatch.go @@ -21,15 +21,15 @@ type dataBatch struct { } func (b *dataBatch) Txs() TxsBatch { - return &txsBatch{Batch: b.Batch} + return &txsBatch{DB: b.DB, Batch: b.Batch} } func (b *dataBatch) Ops() OpsBatch { - return &opsBatch{Batch: b.Batch} + return &opsBatch{DB: b.DB, Batch: b.Batch} } func (b *dataBatch) Que() QueBatch { - return &queBatch{Batch: b.Batch} + return &queBatch{DB: b.DB, Batch: b.Batch} } // Delete all transactions, ops, queued items on the given height. diff --git a/interface/store/databatch_test.go b/interface/store/databatch_test.go new file mode 100644 index 0000000..dbae561 --- /dev/null +++ b/interface/store/databatch_test.go @@ -0,0 +1,24 @@ +package store + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDataBatch_DelAll(t *testing.T) { + db, err := NewDataStore("test") + if !assert.NoError(t, err) { + t.FailNow() + } + + batch := db.Batch() + err = batch.DelAll(0) + if !assert.NoError(t, err) { + t.FailNow() + } + err = batch.Commit() + if !assert.NoError(t, err) { + t.FailNow() + } +} From 5c5ed81a1ec12a4056c372b7b2606d845f504159 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 27 Dec 2018 18:16:06 +0800 Subject: [PATCH 67/73] fix blockchain can not reorganize when fork happens issue #52 --- .gitignore | 2 +- blockchain/blockchain.go | 4 +- database/chaindb.go | 130 +++++++++++++++ database/chainstore.go | 13 +- database/defaultdb.go | 86 ---------- database/headersonly.go | 39 ----- database/txsdb.go | 39 ++--- interface/interface_test.go | 2 - interface/spvservice.go | 264 ++++++++++++++++--------------- interface/store/headers.go | 17 +- interface/store/interface.go | 2 + interface/store/txs.go | 48 +++++- spvwallet.go | 228 ++++++++++++++------------ wallet/store/headers/cache.go | 11 +- wallet/store/sqlite/databatch.go | 4 +- wallet/store/sqlite/interface.go | 10 ++ wallet/store/sqlite/txs.go | 59 ++++++- 17 files changed, 532 insertions(+), 426 deletions(-) create mode 100644 database/chaindb.go delete mode 100644 database/defaultdb.go delete mode 100644 database/headersonly.go diff --git a/.gitignore b/.gitignore index 93b87f8..953fa78 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,7 @@ Log *.lock *.json vendor/ -HEADER/ +header/ DATA/ ela-wallet service diff --git a/blockchain/blockchain.go b/blockchain/blockchain.go index d1d33ad..dd5fa48 100644 --- a/blockchain/blockchain.go +++ b/blockchain/blockchain.go @@ -114,10 +114,10 @@ func (b *BlockChain) CommitBlock(block *util.Block) (newTip, reorg bool, newHeig return newTip, reorg, 0, 0, err } - // Rollback block chain to fork point. + // Process block chain reorganize. log.Infof("REORG!!! At block %d, Wiped out %d blocks", bestHeader.Height, bestHeader.Height-commonAncestor.Height) - err = b.db.Rollback(commonAncestor) + err = b.db.ProcessReorganize(commonAncestor, bestHeader, header) if err != nil { return newTip, reorg, 0, 0, err } diff --git a/database/chaindb.go b/database/chaindb.go new file mode 100644 index 0000000..8c338a1 --- /dev/null +++ b/database/chaindb.go @@ -0,0 +1,130 @@ +package database + +import ( + "github.com/elastos/Elastos.ELA.SPV/util" + "github.com/elastos/Elastos.ELA.Utility/common" +) + +type chainDB struct { + h Headers + t TxsDB +} + +// Headers returns the headers database that stored +// all blockchain headers. +func (d *chainDB) Headers() Headers { + return d.h +} + +// CommitBlock save a block into database, returns how many +// false positive transactions are and error. +func (d *chainDB) CommitBlock(block *util.Block, newTip bool) (fps uint32, err error) { + err = d.h.Put(&block.Header, newTip) + if err != nil { + return 0, err + } + + if newTip { + return d.t.PutTxs(block.Transactions, block.Height) + } + + hash := block.Hash() + return 0, d.t.PutForkTxs(block.Transactions, &hash) +} + +// ProcessReorganize switch chain data to the new best chain. +func (d *chainDB) ProcessReorganize(commonAncestor, prevTip, newTip *util.Header) error { + // 1. Move previous main chain data to fork. + root := commonAncestor.Hash() + header := prevTip + hash := header.Hash() + for !hash.IsEqual(root) { + // Move transactions to fork. + txs, err := d.t.GetTxs(header.Height) + if err != nil { + return err + } + err = d.t.PutForkTxs(txs, &hash) + if err != nil { + return err + } + + // Delete transactions from main chain. + err = d.t.DelTxs(header.Height) + if err != nil { + return err + } + + // Move to previous header. + header, err = d.h.GetPrevious(header) + if err != nil { + return err + } + hash = header.Hash() + } + + // 2. Move new best chain data from fork to main chain. + // subChain stores all fork chain headers by their previous hash, so we can + // index next header by previous header hash. + subChain := make(map[common.Uint256]*util.Header) + header = newTip + hash = header.Hash() + for !hash.IsEqual(root) { + // Index header by it's previous header hash. + subChain[header.Previous()] = header + + // Move to previous header. + var err error + header, err = d.h.GetPrevious(header) + if err != nil { + return err + } + hash = header.Hash() + } + + // Connect sub chain to main chain from fork root. It is important to put + // transactions by order, so we can process UTXOs STXOs correctly. + tip := newTip.Hash() + header = subChain[root] + newRoot := header.Hash() + for !newRoot.IsEqual(tip) { + // Put transactions to main chain. + txs, err := d.t.GetForkTxs(&newRoot) + if err != nil { + return err + } + _, err = d.t.PutTxs(txs, header.Height) + if err != nil { + return err + } + + // Move to next header. + header = subChain[newRoot] + newRoot = header.Hash() + } + + // Set new chain tip. + return d.h.Put(newTip, true) +} + +// Clear delete all data in database. +func (d *chainDB) Clear() error { + if err := d.h.Clear(); err != nil { + return err + } + if err := d.t.Clear(); err != nil { + return err + } + return nil +} + +// Close database. +func (d *chainDB) Close() error { + if err := d.h.Close(); err != nil { + return err + } + if err := d.t.Close(); err != nil { + return err + } + return nil +} diff --git a/database/chainstore.go b/database/chainstore.go index a475774..3c8dd54 100644 --- a/database/chainstore.go +++ b/database/chainstore.go @@ -16,15 +16,10 @@ type ChainStore interface { // false positive transactions are and error. CommitBlock(block *util.Block, newTip bool) (fps uint32, err error) - // Rollback delete all transactions after the reorg point, - // it is used when blockchain reorganized. - Rollback(reorg *util.Header) error + // ProcessReorganize switch chain data to the new best chain. + ProcessReorganize(commonAncestor, prevTip, newTip *util.Header) error } -func NewHeadersOnlyChainDB(db Headers) ChainStore { - return &headersOnlyChainDB{db: db} -} - -func NewDefaultChainDB(h Headers, t TxsDB) ChainStore { - return &defaultChainDB{h: h, t: t} +func NewChainDB(h Headers, t TxsDB) ChainStore { + return &chainDB{h: h, t: t} } diff --git a/database/defaultdb.go b/database/defaultdb.go deleted file mode 100644 index a740f0c..0000000 --- a/database/defaultdb.go +++ /dev/null @@ -1,86 +0,0 @@ -package database - -import "github.com/elastos/Elastos.ELA.SPV/util" - -type defaultChainDB struct { - h Headers - t TxsDB -} - -// Headers returns the headers database that stored -// all blockchain headers. -func (d *defaultChainDB) Headers() Headers { - return d.h -} - -// CommitBlock save a block into database, returns how many -// false positive transactions are and error. -func (d *defaultChainDB) CommitBlock(block *util.Block, newTip bool) (fps uint32, err error) { - err = d.h.Put(&block.Header, newTip) - if err != nil { - return 0, err - } - - // We are on a fork chain, do not commit transactions. - if !newTip { - return 0, nil - } - - batch := d.t.Batch() - for _, tx := range block.Transactions { - fp, err := batch.PutTx(tx, block.Height) - if err != nil { - return 0, batch.Rollback() - } - if fp { - fps++ - } - } - - return fps, batch.Commit() -} - -// RollbackTo delete all transactions after the reorg point, -// it is used when blockchain reorganized. -func (d *defaultChainDB) Rollback(reorg *util.Header) error { - // Get current chain tip - best, err := d.h.GetBest() - if err != nil { - return err - } - - batch := d.t.Batch() - for current := best.Height; current > reorg.Height; current-- { - if err := batch.DelTxs(current); err != nil { - return batch.Rollback() - } - } - - if err := batch.Commit(); err != nil { - return err - } - - return d.h.Put(reorg, true) -} - -// Clear delete all data in database. -func (d *defaultChainDB) Clear() error { - if err := d.h.Clear(); err != nil { - return err - } - if err := d.t.Clear(); err != nil { - return err - } - return nil -} - -// Close database. -func (d *defaultChainDB) Close() error { - if err := d.h.Close(); err != nil { - return err - } - if err := d.t.Close(); err != nil { - return err - } - return nil -} diff --git a/database/headersonly.go b/database/headersonly.go deleted file mode 100644 index 511c026..0000000 --- a/database/headersonly.go +++ /dev/null @@ -1,39 +0,0 @@ -package database - -import ( - "github.com/elastos/Elastos.ELA.SPV/util" -) - -type headersOnlyChainDB struct { - db Headers -} - -// Headers returns the headers database that stored -// all blockchain headers. -func (h *headersOnlyChainDB) Headers() Headers { - return h.db -} - -// CommitBlock save a block into database, returns how many -// false positive transactions are and error. -func (h *headersOnlyChainDB) CommitBlock(block *util.Block, newTip bool) (fps uint32, err error) { - return fps, h.db.Put(&block.Header, newTip) -} - -// RollbackTo delete all transactions after the reorg point, -// it is used when blockchain reorganized. -func (h *headersOnlyChainDB) Rollback(reorg *util.Header) error { - // Just do nothing. Headers never removed from database, - // only transactions need to be rollback. - return nil -} - -// Clear delete all data in database. -func (h *headersOnlyChainDB) Clear() error { - return h.db.Clear() -} - -// Close database. -func (h *headersOnlyChainDB) Close() error { - return h.db.Close() -} diff --git a/database/txsdb.go b/database/txsdb.go index aa9b55a..e4727da 100644 --- a/database/txsdb.go +++ b/database/txsdb.go @@ -5,41 +5,30 @@ import ( "github.com/elastos/Elastos.ELA.Utility/common" ) +// TxsDB stores all transactions in main chain and fork chains. type TxsDB interface { // Extend from DB interface DB - // Batch returns a TxBatch instance for transactions batch - // commit, this can get better performance when commit a bunch - // of transactions within a block. - Batch() TxBatch + // PutTxs persists the main chain transactions into database and can be + // queried by GetTxs(height). Returns the false positive transaction count + // and error. + PutTxs(txs []util.Transaction, height uint32) (uint32, error) + + // PutForkTxs persists the fork chain transactions into database with the + // fork block hash and can be queried by GetForkTxs(hash). + PutForkTxs(txs []util.Transaction, hash *common.Uint256) error // HaveTx returns if the transaction already saved in database // by it's id. HaveTx(txId *common.Uint256) (bool, error) - // GetTxs returns all transactions within the given height. - GetTxs(height uint32) ([]*util.Tx, error) - - // RemoveTxs delete all transactions on the given height. Return - // how many transactions are deleted from database. - RemoveTxs(height uint32) (int, error) -} - -type TxBatch interface { - // PutTx add a store transaction operation into batch, and return - // if it is a false positive and error. - PutTx(tx util.Transaction, height uint32) (bool, error) + // GetTxs returns all transactions in main chain within the given height. + GetTxs(height uint32) ([]util.Transaction, error) - // DelTx add a delete transaction operation into batch. - DelTx(txId *common.Uint256) error + // GetForkTxs returns all transactions within the fork block hash. + GetForkTxs(hash *common.Uint256) ([]util.Transaction, error) - // DelTxs add a delete transactions on given height operation. + // DelTxs remove all transactions in main chain within the given height. DelTxs(height uint32) error - - // Rollback cancel all operations in current batch. - Rollback() error - - // Commit the added transactions into database. - Commit() error } diff --git a/interface/interface_test.go b/interface/interface_test.go index 714cc72..1e726c8 100644 --- a/interface/interface_test.go +++ b/interface/interface_test.go @@ -73,8 +73,6 @@ func (l *TxListener) Notify(id common.Uint256, proof bloom.MerkleProof, tx core. l.service.SubmitTransactionReceipt(id, tx.Hash()) } -func (l *TxListener) Rollback(height uint32) {} - func TestGetListenerKey(t *testing.T) { var key1, key2 common.Uint256 listener := &TxListener{ diff --git a/interface/spvservice.go b/interface/spvservice.go index 623656b..7482915 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -75,7 +75,7 @@ func newSpvService(cfg *Config) (*spvservice, error) { listeners: make(map[common.Uint256]TransactionListener), } - chainStore := database.NewDefaultChainDB(headerStore, service) + chainStore := database.NewChainDB(headerStore, service) serviceCfg := &sdk.Config{ DataDir: dataDir, @@ -190,16 +190,88 @@ func (s *spvservice) GetFilterData() ([]*common.Uint168, []*util.OutPoint) { return s.db.Addrs().GetAll(), ops } -// Batch returns a TxBatch instance for transactions batch -// commit, this can get better performance when commit a bunch -// of transactions within a block. -func (s *spvservice) Batch() database.TxBatch { - return &txBatch{ - db: s.db, - batch: s.db.Batch(), - rollback: s.rollback, - listeners: s.listeners, +func (s *spvservice) putTx(batch store.DataBatch, utx util.Transaction, + height uint32) (bool, error) { + + tx := utx.(*iutil.Tx) + hits := make(map[common.Uint168]struct{}) + ops := make(map[*util.OutPoint]common.Uint168) + for index, output := range tx.Outputs { + if s.db.Addrs().GetFilter().ContainAddr(output.ProgramHash) { + outpoint := util.NewOutPoint(tx.Hash(), uint16(index)) + ops[outpoint] = output.ProgramHash + hits[output.ProgramHash] = struct{}{} + } + } + + for _, input := range tx.Inputs { + op := input.Previous + addr := s.db.Ops().HaveOp(util.NewOutPoint(op.TxID, op.Index)) + if addr != nil { + hits[*addr] = struct{}{} + } + } + + if len(hits) == 0 { + return true, nil } + + for op, addr := range ops { + if err := batch.Ops().Put(op, addr); err != nil { + return false, err + } + } + + for _, listener := range s.listeners { + hash, _ := common.Uint168FromAddress(listener.Address()) + if _, ok := hits[*hash]; ok { + // skip transactions that not match the require type + if listener.Type() != tx.TxType { + continue + } + + // queue message + batch.Que().Put(&store.QueItem{ + NotifyId: getListenerKey(listener), + TxId: tx.Hash(), + Height: height, + }) + } + } + + return false, batch.Txs().Put(util.NewTx(utx, height)) +} + +// PutTxs persists the main chain transactions into database and can be +// queried by GetTxs(height). Returns the false positive transaction count +// and error. +func (s *spvservice) PutTxs(txs []util.Transaction, height uint32) (uint32, error) { + fps := uint32(0) + batch := s.db.Batch() + defer batch.Rollback() + for _, tx := range txs { + fp, err := s.putTx(batch, tx, height) + if err != nil { + return 0, err + } + if fp { + fps++ + } + } + if err := batch.Commit(); err != nil { + return 0, err + } + return fps, nil +} + +// PutForkTxs persists the fork chain transactions into database with the +// fork block hash and can be queried by GetForkTxs(hash). +func (s *spvservice) PutForkTxs(txs []util.Transaction, hash *common.Uint256) error { + ftxs := make([]*util.Tx, 0, len(txs)) + for _, utx := range txs { + ftxs = append(ftxs, util.NewTx(utx, 0)) + } + return s.db.Txs().PutForkTxs(ftxs, hash) } // HaveTx returns if the transaction already saved in database @@ -209,19 +281,65 @@ func (s *spvservice) HaveTx(txId *common.Uint256) (bool, error) { return tx != nil, err } -// GetTxs returns all transactions within the given height. -func (s *spvservice) GetTxs(height uint32) ([]*util.Tx, error) { - return nil, nil +// GetTxs returns all transactions in main chain within the given height. +func (s *spvservice) GetTxs(height uint32) ([]util.Transaction, error) { + txIDs, err := s.db.Txs().GetIds(height) + if err != nil { + return nil, err + } + + txs := make([]util.Transaction, 0, len(txIDs)) + for _, txID := range txIDs { + tx := newTransaction() + utx, err := s.db.Txs().Get(txID) + if err != nil { + return nil, err + } + err = tx.Deserialize(bytes.NewReader(utx.RawData)) + if err != nil { + return nil, err + } + txs = append(txs, tx) + } + return txs, nil +} + +// GetForkTxs returns all transactions within the fork block hash. +func (s *spvservice) GetForkTxs(hash *common.Uint256) ([]util.Transaction, error) { + ftxs, err := s.db.Txs().GetForkTxs(hash) + if err != nil { + return nil, err + } + + txs := make([]util.Transaction, 0, len(ftxs)) + for _, ftx := range ftxs { + tx := newTransaction() + err = tx.Deserialize(bytes.NewReader(ftx.RawData)) + if err != nil { + return nil, err + } + txs = append(txs, tx) + } + return txs, nil } -// RemoveTxs delete all transactions on the given height. Return -// how many transactions are deleted from database. -func (s *spvservice) RemoveTxs(height uint32) (int, error) { +// DelTxs remove all transactions in main chain within the given height. +func (s *spvservice) DelTxs(height uint32) error { + // Delete transactions, outpoints and queued items. batch := s.db.Batch() + defer batch.Rollback() if err := batch.DelAll(height); err != nil { - return 0, batch.Rollback() + return err } - return 0, batch.Commit() + if err := batch.Commit(); err != nil { + return err + } + + // Invoke main chain rollback. + if s.rollback != nil { + s.rollback(height) + } + return nil } // TransactionAnnounce will be invoked when received a new announced transaction. @@ -384,113 +502,3 @@ func getConfirmations(tx core.Transaction) uint32 { } return DefaultConfirmations } - -type txBatch struct { - db store.DataStore - batch store.DataBatch - heights []uint32 - rollback func(height uint32) - listeners map[common.Uint256]TransactionListener -} - -// PutTx add a store transaction operation into batch, and return -// if it is a false positive and error. -func (b *txBatch) PutTx(utx util.Transaction, height uint32) (bool, error) { - tx := utx.(*iutil.Tx) - hits := make(map[common.Uint168]struct{}) - ops := make(map[*util.OutPoint]common.Uint168) - for index, output := range tx.Outputs { - if b.db.Addrs().GetFilter().ContainAddr(output.ProgramHash) { - outpoint := util.NewOutPoint(tx.Hash(), uint16(index)) - ops[outpoint] = output.ProgramHash - hits[output.ProgramHash] = struct{}{} - } - } - - for _, input := range tx.Inputs { - op := input.Previous - addr := b.db.Ops().HaveOp(util.NewOutPoint(op.TxID, op.Index)) - if addr != nil { - hits[*addr] = struct{}{} - } - } - - if len(hits) == 0 { - return true, nil - } - - for op, addr := range ops { - if err := b.batch.Ops().Put(op, addr); err != nil { - return false, err - } - } - - for _, listener := range b.listeners { - hash, _ := common.Uint168FromAddress(listener.Address()) - if _, ok := hits[*hash]; ok { - // skip transactions that not match the require type - if listener.Type() != tx.TxType { - continue - } - - // queue message - b.batch.Que().Put(&store.QueItem{ - NotifyId: getListenerKey(listener), - TxId: tx.Hash(), - Height: height, - }) - } - } - - return false, b.batch.Txs().Put(util.NewTx(utx, height)) -} - -// DelTx add a delete transaction operation into batch. -func (b *txBatch) DelTx(txId *common.Uint256) error { - utx, err := b.db.Txs().Get(txId) - if err != nil { - return err - } - - var tx core.Transaction - err = tx.Deserialize(bytes.NewReader(utx.RawData)) - if err != nil { - return err - } - - for index := range tx.Outputs { - outpoint := util.NewOutPoint(utx.Hash, uint16(index)) - b.batch.Ops().Del(outpoint) - } - - return b.batch.Txs().Del(txId) -} - -// DelTxs add a delete transactions on given height operation. -func (b *txBatch) DelTxs(height uint32) error { - if b.rollback != nil { - b.heights = append(b.heights, height) - } - return b.batch.DelAll(height) -} - -// Rollback cancel all operations in current batch. -func (b *txBatch) Rollback() error { - return b.batch.Rollback() -} - -// Commit the added transactions into database. -func (b *txBatch) Commit() error { - err := b.batch.Commit() - if err != nil { - return err - } - - go func(heights []uint32) { - for _, height := range heights { - b.rollback(height) - } - }(b.heights) - - return nil -} diff --git a/interface/store/headers.go b/interface/store/headers.go index 570597f..6799d2c 100644 --- a/interface/store/headers.go +++ b/interface/store/headers.go @@ -64,7 +64,7 @@ func (h *headers) initCache() { headers = append(headers, sh) } for i := len(headers) - 1; i >= 0; i-- { - h.cache.Set(headers[i]) + h.cache.set(headers[i]) } } @@ -72,7 +72,7 @@ func (h *headers) Put(header *util.Header, newTip bool) error { h.Lock() defer h.Unlock() - h.cache.Set(header) + h.cache.set(header) if newTip { h.cache.tip = header } @@ -110,7 +110,7 @@ func (h *headers) Get(hash *common.Uint256) (header *util.Header, err error) { h.RLock() defer h.RUnlock() - header, err = h.cache.Get(hash) + header, err = h.cache.get(hash) if err == nil { return header, nil } @@ -186,7 +186,6 @@ func (h *headers) getHeader(key []byte) (*util.Header, error) { } type cache struct { - sync.RWMutex size int tip *util.Header headers *ordered_map.OrderedMap @@ -207,20 +206,14 @@ func (cache *cache) pop() { } } -func (cache *cache) Set(header *util.Header) { - cache.Lock() - defer cache.Unlock() - +func (cache *cache) set(header *util.Header) { if cache.headers.Len() > cache.size { cache.pop() } cache.headers.Set(header.Hash().String(), header) } -func (cache *cache) Get(hash *common.Uint256) (*util.Header, error) { - cache.RLock() - defer cache.RUnlock() - +func (cache *cache) get(hash *common.Uint256) (*util.Header, error) { sh, ok := cache.headers.Get(hash.String()) if !ok { return nil, errors.New("Header not found in cache ") diff --git a/interface/store/interface.go b/interface/store/interface.go index 48c2b35..dd24fc0 100644 --- a/interface/store/interface.go +++ b/interface/store/interface.go @@ -52,6 +52,8 @@ type Txs interface { Get(txId *common.Uint256) (*util.Tx, error) GetAll() ([]*util.Tx, error) GetIds(height uint32) ([]*common.Uint256, error) + PutForkTxs(txs []*util.Tx, hash *common.Uint256) error + GetForkTxs(hash *common.Uint256) ([]*util.Tx, error) Del(txId *common.Uint256) error Batch() TxsBatch } diff --git a/interface/store/txs.go b/interface/store/txs.go index 86d7a5d..1ed73f3 100644 --- a/interface/store/txs.go +++ b/interface/store/txs.go @@ -15,6 +15,7 @@ import ( var ( BKTTxs = []byte("T") BKTHeightTxs = []byte("H") + BKTForkTxs = []byte("F") ) // Ensure txs implement Txs interface. @@ -102,13 +103,54 @@ func getTxIds(data []byte) (txIds []*common.Uint256) { data = data[2:] for i := uint16(0); i < count; i++ { var txId common.Uint256 - copy(txId[:], data[i*common.UINT256SIZE:(i+1)*common.UINT256SIZE]) + copy(txId[:], data[i*32:(i+1)*32]) txIds = append(txIds, &txId) } return txIds } +func (t *txs) PutForkTxs(txs []*util.Tx, hash *common.Uint256) error { + t.Lock() + defer t.Unlock() + + buf := new(bytes.Buffer) + if err := common.WriteUint16(buf, uint16(len(txs))); err != nil { + return err + } + for _, tx := range txs { + if err := tx.Serialize(buf); err != nil { + return err + } + } + return t.db.Put(toKey(BKTForkTxs, hash.Bytes()...), buf.Bytes(), nil) +} + +func (t *txs) GetForkTxs(hash *common.Uint256) ([]*util.Tx, error) { + t.RLock() + defer t.RUnlock() + + data, err := t.db.Get(toKey(BKTForkTxs, hash.Bytes()...), nil) + if err != nil { + return nil, err + } + buf := bytes.NewReader(data) + count, err := common.ReadUint16(buf) + if err != nil { + return nil, err + } + + txs := make([]*util.Tx, count) + for i := range txs { + var utx util.Tx + if err := utx.Deserialize(buf); err != nil { + return nil, err + } + txs[i] = &utx + } + return txs, nil +} + func (t *txs) GetAll() (txs []*util.Tx, err error) { t.RLock() defer t.RUnlock() @@ -163,9 +205,9 @@ func delTxId(data []byte, hash *common.Uint256) []byte { data = data[2:] for i := uint16(0); i < count; i++ { var txId common.Uint256 - copy(txId[:], data[i*common.UINT256SIZE:(i+1)*common.UINT256SIZE]) + copy(txId[:], data[i*32:(i+1)*32]) if txId.IsEqual(*hash) { - data = append(data[0:i*common.UINT256SIZE], data[(i+1)*common.UINT256SIZE:]...) + data = append(data[0:i*32], data[(i+1)*32:]...) break } } diff --git a/spvwallet.go b/spvwallet.go index dc541f7..7dd6d30 100644 --- a/spvwallet.go +++ b/spvwallet.go @@ -32,15 +32,91 @@ type spvwallet struct { filter *sdk.AddrFilter } -// Batch returns a TxBatch instance for transactions batch -// commit, this can get better performance when commit a bunch -// of transactions within a block. -func (w *spvwallet) Batch() database.TxBatch { - return &txBatch{ - db: w.db, - batch: w.db.Batch(), - filter: w.getAddrFilter(), +func (w *spvwallet) putTx(batch sqlite.DataBatch, utx util.Transaction, + height uint32) (bool, error) { + + tx := utx.(*sutil.Tx) + txId := tx.Hash() + hits := 0 + + // Check if any UTXOs within this wallet have been spent. + for _, input := range tx.Inputs { + // Move UTXO to STXO + op := util.NewOutPoint(input.Previous.TxID, input.Previous.Index) + utxo, _ := w.db.UTXOs().Get(op) + // Skip if no match. + if utxo == nil { + continue + } + + err := batch.STXOs().Put(sutil.NewSTXO(utxo, height, txId)) + if err != nil { + return false, nil + } + hits++ + } + + // Check if there are any output to this wallet address. + for index, output := range tx.Outputs { + // Filter address + if w.getAddrFilter().ContainAddr(output.ProgramHash) { + var lockTime = output.OutputLock + if tx.TxType == core.CoinBase { + lockTime = height + 100 + } + utxo := sutil.NewUTXO(txId, height, index, output.Value, lockTime, output.ProgramHash) + err := batch.UTXOs().Put(utxo) + if err != nil { + return false, err + } + hits++ + } } + + // If no hits, no need to save transaction + if hits == 0 { + return true, nil + } + + // Save transaction + err := batch.Txs().Put(util.NewTx(tx, height)) + if err != nil { + return false, err + } + + return false, nil +} + +// PutTxs persists the main chain transactions into database and can be +// queried by GetTxs(height). Returns the false positive transaction count +// and error. +func (w *spvwallet) PutTxs(txs []util.Transaction, height uint32) (uint32, error) { + fps := uint32(0) + batch := w.db.Batch() + defer batch.Rollback() + for _, tx := range txs { + fp, err := w.putTx(batch, tx, height) + if err != nil { + return 0, err + } + if fp { + fps++ + } + } + if err := batch.Commit(); err != nil { + return 0, err + } + return fps, nil +} + +// PutForkTxs persists the fork chain transactions into database with the +// fork block hash and can be queried by GetForkTxs(hash). +func (w *spvwallet) PutForkTxs(txs []util.Transaction, hash *common.Uint256) error { + ftxs := make([]*util.Tx, 0, len(txs)) + for _, tx := range txs { + ftxs = append(ftxs, util.NewTx(tx, 0)) + } + return w.db.Txs().PutForkTxs(ftxs, hash) } // HaveTx returns if the transaction already saved in database @@ -50,20 +126,50 @@ func (w *spvwallet) HaveTx(txId *common.Uint256) (bool, error) { return tx != nil, err } -// GetTxs returns all transactions within the given height. -func (w *spvwallet) GetTxs(height uint32) ([]*util.Tx, error) { - return nil, nil +// GetTxs returns all transactions in main chain within the given height. +func (w *spvwallet) GetTxs(height uint32) ([]util.Transaction, error) { + txs, err := w.db.Txs().GetAllFrom(height) + if err != nil { + return nil, err + } + + utxs := make([]util.Transaction, 0, len(txs)) + for _, tx := range txs { + wtx := newTransaction() + if err := wtx.Deserialize(bytes.NewReader(tx.RawData)); err != nil { + return nil, err + } + utxs = append(utxs, wtx) + } + return utxs, nil } -// RemoveTxs delete all transactions on the given height. Return -// how many transactions are deleted from database. -func (w *spvwallet) RemoveTxs(height uint32) (int, error) { - batch := w.db.Batch() - err := batch.RollbackHeight(height) +// GetForkTxs returns all transactions within the fork block hash. +func (w *spvwallet) GetForkTxs(hash *common.Uint256) ([]util.Transaction, error) { + ftxs, err := w.db.Txs().GetForkTxs(hash) if err != nil { - return 0, batch.Rollback() + return nil, err } - return 0, batch.Commit() + + txs := make([]util.Transaction, 0, len(ftxs)) + for _, ftx := range ftxs { + tx := newTransaction() + if err := tx.Deserialize(bytes.NewReader(ftx.RawData)); err != nil { + return nil, err + } + txs = append(txs, tx) + } + return txs, nil +} + +// DelTxs remove all transactions in main chain within the given height. +func (w *spvwallet) DelTxs(height uint32) error { + batch := w.db.Batch() + defer batch.Rollback() + if err := batch.RollbackHeight(height); err != nil { + return err + } + return batch.Commit() } // Clear delete all data in database. @@ -154,90 +260,6 @@ func (w *spvwallet) BlockCommitted(block *util.Block) { // TODO } -type txBatch struct { - db sqlite.DataStore - batch sqlite.DataBatch - filter *sdk.AddrFilter -} - -// PutTx add a store transaction operation into batch, and return -// if it is a false positive and error. -func (b *txBatch) PutTx(utx util.Transaction, height uint32) (bool, error) { - tx := utx.(*sutil.Tx) - txId := tx.Hash() - hits := 0 - - // Check if any UTXOs within this wallet have been spent. - for _, input := range tx.Inputs { - // Move UTXO to STXO - op := util.NewOutPoint(input.Previous.TxID, input.Previous.Index) - utxo, _ := b.db.UTXOs().Get(op) - // Skip if no match. - if utxo == nil { - continue - } - - err := b.batch.STXOs().Put(sutil.NewSTXO(utxo, height, txId)) - if err != nil { - return false, nil - } - hits++ - } - - // Check if there are any output to this wallet address. - for index, output := range tx.Outputs { - // Filter address - if b.filter.ContainAddr(output.ProgramHash) { - var lockTime = output.OutputLock - if tx.TxType == core.CoinBase { - lockTime = height + 100 - } - utxo := sutil.NewUTXO(txId, height, index, output.Value, lockTime, output.ProgramHash) - err := b.batch.UTXOs().Put(utxo) - if err != nil { - return false, err - } - hits++ - } - } - - // If no hits, no need to save transaction - if hits == 0 { - return true, nil - } - - // Save transaction - err := b.batch.Txs().Put(util.NewTx(tx, height)) - if err != nil { - return false, err - } - - return false, nil -} - -// DelTx add a delete transaction operation into batch. -func (b *txBatch) DelTx(txId *common.Uint256) error { - return b.batch.Txs().Del(txId) -} - -// DelTxs add a delete transactions on given height operation. -func (b *txBatch) DelTxs(height uint32) error { - // Delete transactions is used when blockchain doing rollback, this not - // only delete the transactions on the given height, and also restore - // STXOs and remove UTXOs within these transactions. - return b.batch.RollbackHeight(height) -} - -// Rollback cancel all operations in current batch. -func (b *txBatch) Rollback() error { - return b.batch.Rollback() -} - -// Commit the added transactions into database. -func (b *txBatch) Commit() error { - return b.batch.Commit() -} - // Functions for RPC service. func (w *spvwallet) notifyNewAddress(params httputil.Params) (interface{}, error) { addrStr, ok := params.String("addr") @@ -296,7 +318,7 @@ func NewWallet() (*spvwallet, error) { w := spvwallet{ db: db, } - chainStore := database.NewDefaultChainDB(headers, &w) + chainStore := database.NewChainDB(headers, &w) // Initialize spv service w.IService, err = sdk.NewService( diff --git a/wallet/store/headers/cache.go b/wallet/store/headers/cache.go index 1160b7e..03e5ce0 100644 --- a/wallet/store/headers/cache.go +++ b/wallet/store/headers/cache.go @@ -2,15 +2,14 @@ package headers import ( "errors" - "sync" - "github.com/cevaris/ordered_map" "github.com/elastos/Elastos.ELA.SPV/util" + + "github.com/cevaris/ordered_map" "github.com/elastos/Elastos.ELA.Utility/common" ) type cache struct { - sync.RWMutex size int tip *util.Header headers *ordered_map.OrderedMap @@ -32,9 +31,6 @@ func (cache *cache) pop() { } func (cache *cache) set(header *util.Header) { - cache.Lock() - defer cache.Unlock() - if cache.headers.Len() > cache.size { cache.pop() } @@ -42,9 +38,6 @@ func (cache *cache) set(header *util.Header) { } func (cache *cache) get(hash *common.Uint256) (*util.Header, error) { - cache.RLock() - defer cache.RUnlock() - sh, ok := cache.headers.Get(hash.String()) if !ok { return nil, errors.New("Header not found in cache ") diff --git a/wallet/store/sqlite/databatch.go b/wallet/store/sqlite/databatch.go index 84d6f07..5a3ff30 100644 --- a/wallet/store/sqlite/databatch.go +++ b/wallet/store/sqlite/databatch.go @@ -64,8 +64,8 @@ func (d *dataBatch) RollbackHeight(height uint32) error { } // Rollback STXOs, move UTXOs back first, then delete the STXOs - _, err = d.Exec(`INSERT OR REPLACE INTO UTXOs(OutPoint, Value, LockTime, AtHeight, ScriptHash) - SELECT OutPoint, Value, LockTime, AtHeight, ScriptHash FROM STXOs WHERE SpendHeight=?`, height) + _, err = d.Exec(`INSERT OR REPLACE INTO UTXOs(OutPoint, Value, LockTime, AtHeight, Address) + SELECT OutPoint, Value, LockTime, AtHeight, Address FROM STXOs WHERE SpendHeight=?`, height) if err != nil { return err } diff --git a/wallet/store/sqlite/interface.go b/wallet/store/sqlite/interface.go index 3e4167c..2667ac1 100644 --- a/wallet/store/sqlite/interface.go +++ b/wallet/store/sqlite/interface.go @@ -77,12 +77,22 @@ type Txs interface { // Fetch all transactions from database GetAll() ([]*util.Tx, error) + // Fetch all transactions from the given height. + GetAllFrom(height uint32) ([]*util.Tx, error) + // Fetch all unconfirmed transactions. GetAllUnconfirmed() ([]*util.Tx, error) // Delete a transaction from the db Del(txId *common.Uint256) error + // PutForkTxs persists the fork chain transactions into database with the + // fork block hash and can be queried by GetForkTxs(hash). + PutForkTxs(txs []*util.Tx, hash *common.Uint256) error + + // GetForkTxs returns all transactions within the fork block hash. + GetForkTxs(hash *common.Uint256) ([]*util.Tx, error) + // Batch return a TxsBatch Batch() TxsBatch } diff --git a/wallet/store/sqlite/txs.go b/wallet/store/sqlite/txs.go index cef42a1..474879f 100644 --- a/wallet/store/sqlite/txs.go +++ b/wallet/store/sqlite/txs.go @@ -1,6 +1,7 @@ package sqlite import ( + "bytes" "database/sql" "math" "sync" @@ -12,11 +13,16 @@ import ( ) const CreateTxsDB = `CREATE TABLE IF NOT EXISTS Txs( - Hash BLOB NOT NULL PRIMARY KEY, - Height INTEGER NOT NULL, - Timestamp INTEGER NOT NULL, - RawData BLOB NOT NULL - );` + Hash BLOB NOT NULL PRIMARY KEY, + Height INTEGER NOT NULL, + Timestamp INTEGER NOT NULL, + RawData BLOB NOT NULL + ); + CREATE TABLE IF NOT EXISTS ForkTxs( + Hash BLOB NOT NULL PRIMARY KEY, + Num INTEGER NOT NULL, + RawData BLOB NOT NULL + );` // Ensure txs implement Txs interface. var _ Txs = (*txs)(nil) @@ -119,6 +125,49 @@ func (t *txs) Del(txId *common.Uint256) error { return err } +// PutForkTxs persists the fork chain transactions into database with the +// fork block hash and can be queried by GetForkTxs(hash). +func (t *txs) PutForkTxs(txs []*util.Tx, hash *common.Uint256) error { + t.Lock() + defer t.Unlock() + + buf := new(bytes.Buffer) + for _, tx := range txs { + if err := tx.Serialize(buf); err != nil { + return err + } + } + + _, err := t.Exec(`INSERT OR REPLACE INTO ForkTxs(Hash, Num, RawData) VALUES(?,?,?)`, + hash.Bytes(), len(txs), buf.Bytes()) + return err +} + +// GetForkTxs returns all transactions within the fork block hash. +func (t *txs) GetForkTxs(hash *common.Uint256) ([]*util.Tx, error) { + t.RLock() + defer t.RUnlock() + + row := t.QueryRow(`SELECT Num, RawData FROM ForkTxs WHERE Hash=?`, + hash.Bytes()) + var num int + var data []byte + if err := row.Scan(&num, &data); err != nil { + return nil, err + } + + txs := make([]*util.Tx, num) + buf := bytes.NewReader(data) + for i := range txs { + var utx util.Tx + if err := utx.Deserialize(buf); err != nil { + return nil, err + } + txs[i] = &utx + } + return txs, nil +} + func (t *txs) Batch() TxsBatch { t.Lock() defer t.Unlock() From 861c803411199c20042327ac6395be8df6aefe1a Mon Sep 17 00:00:00 2001 From: AlexPan Date: Thu, 3 Jan 2019 12:30:24 +0800 Subject: [PATCH 68/73] move common/ crypto/ p2p/ dependency from Utility to ELA project issue #55 --- blockchain/blockchain.go | 2 +- blockchain/difficulty.go | 2 +- bloom/filter.go | 4 +- bloom/merkleblock.go | 4 +- bloom/merklebranch.go | 4 +- bloom/merklebranch_test.go | 8 +-- bloom/merkleproof.go | 4 +- client.go | 23 ++++---- config.go | 2 +- database/chaindb.go | 3 +- database/headers.go | 2 +- database/txsdb.go | 3 +- interface/config.go | 56 ++++++++++--------- interface/interface.go | 14 ++--- interface/interface_test.go | 26 ++++----- interface/iutil/header.go | 8 +-- interface/iutil/tx.go | 6 +- interface/keystore.go | 2 +- interface/spvservice.go | 24 ++++---- interface/store/addrs.go | 2 +- interface/store/databatch.go | 6 +- interface/store/headers.go | 2 +- interface/store/interface.go | 2 +- interface/store/ops.go | 2 +- interface/store/opsbatch.go | 2 +- interface/store/que.go | 2 +- interface/store/que_test.go | 2 +- interface/store/quebatch.go | 2 +- interface/store/txs.go | 2 +- interface/store/txs_test.go | 2 +- interface/store/txsbatch.go | 2 +- log.go | 6 +- peer/log.go | 2 +- peer/peer.go | 8 +-- sdk/account.go | 29 +++++++--- sdk/addrfilter.go | 2 +- sdk/crypto.go | 2 +- sdk/interface.go | 2 +- sdk/protocal.go | 2 +- sdk/service.go | 10 ++-- spvwallet.go | 56 ++++++++++--------- sync/manager.go | 4 +- util/header.go | 4 +- util/interface.go | 2 +- util/outpoint.go | 2 +- util/tx.go | 4 +- wallet/client.go | 2 +- wallet/client/account/account.go | 4 +- wallet/client/common.go | 5 +- wallet/client/database/database.go | 2 +- wallet/client/database/interface.go | 2 +- wallet/client/keystore.go | 11 ++-- wallet/client/keystore_file.go | 5 +- wallet/client/transaction/transaction.go | 26 ++++----- wallet/client/wallet.go | 70 ++++++++++++------------ wallet/store/headers/cache.go | 2 +- wallet/store/headers/database.go | 2 +- wallet/store/sqlite/addrs.go | 2 +- wallet/store/sqlite/addrsbatch.go | 2 +- wallet/store/sqlite/interface.go | 2 +- wallet/store/sqlite/stxos.go | 2 +- wallet/store/sqlite/txs.go | 2 +- wallet/store/sqlite/txsbatch.go | 2 +- wallet/store/sqlite/utxos.go | 2 +- wallet/sutil/addr.go | 2 +- wallet/sutil/header.go | 10 ++-- wallet/sutil/stxo.go | 2 +- wallet/sutil/tx.go | 6 +- wallet/sutil/utxo.go | 2 +- 69 files changed, 274 insertions(+), 253 deletions(-) diff --git a/blockchain/blockchain.go b/blockchain/blockchain.go index dd5fa48..c9885b5 100644 --- a/blockchain/blockchain.go +++ b/blockchain/blockchain.go @@ -8,7 +8,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) const ( diff --git a/blockchain/difficulty.go b/blockchain/difficulty.go index 0e7ade8..2bda814 100644 --- a/blockchain/difficulty.go +++ b/blockchain/difficulty.go @@ -4,7 +4,7 @@ import ( "math/big" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) var PowLimit = new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 255), big.NewInt(1)) diff --git a/bloom/filter.go b/bloom/filter.go index 9dfc519..68d7b61 100644 --- a/bloom/filter.go +++ b/bloom/filter.go @@ -6,8 +6,8 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/p2p/msg" ) const ( diff --git a/bloom/merkleblock.go b/bloom/merkleblock.go index e8d662e..f526a80 100644 --- a/bloom/merkleblock.go +++ b/bloom/merkleblock.go @@ -6,8 +6,8 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/p2p/msg" ) // mBlock is used to house intermediate information needed to generate a diff --git a/bloom/merklebranch.go b/bloom/merklebranch.go index 737687e..55213a0 100644 --- a/bloom/merklebranch.go +++ b/bloom/merklebranch.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/p2p/msg" ) type MerkleBranch struct { diff --git a/bloom/merklebranch_test.go b/bloom/merklebranch_test.go index 07175f6..8c715bb 100644 --- a/bloom/merklebranch_test.go +++ b/bloom/merklebranch_test.go @@ -6,10 +6,10 @@ import ( "os" "testing" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" "github.com/elastos/Elastos.ELA/auxpow" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/types" + "github.com/elastos/Elastos.ELA/p2p/msg" ) func TestMerkleBlock_GetTxMerkleBranch(t *testing.T) { @@ -48,7 +48,7 @@ func run(txs uint32) { merkleRoot := *mBlock.calcHash(treeDepth(txs), 0) // Create and return the merkle block. merkleBlock := msg.MerkleBlock{ - Header: &core.Header{ + Header: &types.Header{ MerkleRoot: merkleRoot, }, Transactions: mBlock.NumTx, diff --git a/bloom/merkleproof.go b/bloom/merkleproof.go index 75e9605..8f45cfd 100644 --- a/bloom/merkleproof.go +++ b/bloom/merkleproof.go @@ -4,8 +4,8 @@ import ( "fmt" "io" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/p2p/msg" ) // maxFlagsPerMerkleProof is the maximum number of flag bytes that could diff --git a/client.go b/client.go index 103ad92..12c5962 100644 --- a/client.go +++ b/client.go @@ -4,8 +4,11 @@ import ( "fmt" "github.com/elastos/Elastos.ELA.SPV/wallet" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" + + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/contract/program" + "github.com/elastos/Elastos.ELA/core/types" + "github.com/elastos/Elastos.ELA/core/types/payload" ) var Version string @@ -16,11 +19,11 @@ func main() { } func getSystemAssetId() common.Uint256 { - systemToken := &core.Transaction{ - TxType: core.RegisterAsset, + systemToken := &types.Transaction{ + TxType: types.RegisterAsset, PayloadVersion: 0, - Payload: &core.PayloadRegisterAsset{ - Asset: core.Asset{ + Payload: &payload.PayloadRegisterAsset{ + Asset: payload.Asset{ Name: "ELA", Precision: 0x08, AssetType: 0x00, @@ -28,10 +31,10 @@ func getSystemAssetId() common.Uint256 { Amount: 0 * 100000000, Controller: common.Uint168{}, }, - Attributes: []*core.Attribute{}, - Inputs: []*core.Input{}, - Outputs: []*core.Output{}, - Programs: []*core.Program{}, + Attributes: []*types.Attribute{}, + Inputs: []*types.Input{}, + Outputs: []*types.Output{}, + Programs: []*program.Program{}, } return systemToken.Hash() } diff --git a/config.go b/config.go index c4205d5..de3b00c 100644 --- a/config.go +++ b/config.go @@ -8,7 +8,7 @@ import ( "net" "os" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) const ( diff --git a/database/chaindb.go b/database/chaindb.go index 8c338a1..79f23ba 100644 --- a/database/chaindb.go +++ b/database/chaindb.go @@ -2,7 +2,8 @@ package database import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + + "github.com/elastos/Elastos.ELA/common" ) type chainDB struct { diff --git a/database/headers.go b/database/headers.go index e2b44ea..c7be5be 100644 --- a/database/headers.go +++ b/database/headers.go @@ -3,7 +3,7 @@ package database import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) type Headers interface { diff --git a/database/txsdb.go b/database/txsdb.go index e4727da..7ce4783 100644 --- a/database/txsdb.go +++ b/database/txsdb.go @@ -2,7 +2,8 @@ package database import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + + "github.com/elastos/Elastos.ELA/common" ) // TxsDB stores all transactions in main chain and fork chains. diff --git a/interface/config.go b/interface/config.go index 148036f..76969bf 100644 --- a/interface/config.go +++ b/interface/config.go @@ -6,17 +6,19 @@ import ( "github.com/elastos/Elastos.ELA.SPV/interface/iutil" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/crypto" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/contract/program" + "github.com/elastos/Elastos.ELA/core/types" + "github.com/elastos/Elastos.ELA/core/types/payload" + "github.com/elastos/Elastos.ELA/crypto" ) func newBlockHeader() util.BlockHeader { - return iutil.NewHeader(&core.Header{}) + return iutil.NewHeader(&types.Header{}) } func newTransaction() util.Transaction { - return iutil.NewTx(&core.Transaction{}) + return iutil.NewTx(&types.Transaction{}) } // GenesisHeader creates a specific genesis header by the given @@ -26,22 +28,22 @@ func GenesisHeader(foundation *common.Uint168) util.BlockHeader { genesisTime := time.Date(2017, time.December, 22, 10, 0, 0, 0, time.UTC) // header - header := core.Header{ - Version: core.BlockVersion, + header := types.Header{ + Version: 0, Previous: common.EmptyHash, MerkleRoot: common.EmptyHash, Timestamp: uint32(genesisTime.Unix()), Bits: 0x1d03ffff, - Nonce: core.GenesisNonce, + Nonce: types.GenesisNonce, Height: uint32(0), } // ELA coin - elaCoin := &core.Transaction{ - TxType: core.RegisterAsset, + elaCoin := &types.Transaction{ + TxType: types.RegisterAsset, PayloadVersion: 0, - Payload: &core.PayloadRegisterAsset{ - Asset: core.Asset{ + Payload: &payload.PayloadRegisterAsset{ + Asset: payload.Asset{ Name: "ELA", Precision: 0x08, AssetType: 0x00, @@ -49,31 +51,31 @@ func GenesisHeader(foundation *common.Uint168) util.BlockHeader { Amount: 0 * 100000000, Controller: common.Uint168{}, }, - Attributes: []*core.Attribute{}, - Inputs: []*core.Input{}, - Outputs: []*core.Output{}, - Programs: []*core.Program{}, + Attributes: []*types.Attribute{}, + Inputs: []*types.Input{}, + Outputs: []*types.Output{}, + Programs: []*program.Program{}, } - coinBase := &core.Transaction{ - TxType: core.CoinBase, - PayloadVersion: core.PayloadCoinBaseVersion, - Payload: new(core.PayloadCoinBase), - Inputs: []*core.Input{ + coinBase := &types.Transaction{ + TxType: types.CoinBase, + PayloadVersion: payload.PayloadCoinBaseVersion, + Payload: new(payload.PayloadCoinBase), + Inputs: []*types.Input{ { - Previous: core.OutPoint{ + Previous: types.OutPoint{ TxID: common.EmptyHash, Index: 0x0000, }, Sequence: 0x00000000, }, }, - Attributes: []*core.Attribute{}, + Attributes: []*types.Attribute{}, LockTime: 0, - Programs: []*core.Program{}, + Programs: []*program.Program{}, } - coinBase.Outputs = []*core.Output{ + coinBase.Outputs = []*types.Output{ { AssetID: elaCoin.Hash(), Value: 3300 * 10000 * 100000000, @@ -82,10 +84,10 @@ func GenesisHeader(foundation *common.Uint168) util.BlockHeader { } nonce := []byte{0x4d, 0x65, 0x82, 0x21, 0x07, 0xfc, 0xfd, 0x52} - txAttr := core.NewAttribute(core.Nonce, nonce) + txAttr := types.NewAttribute(types.Nonce, nonce) coinBase.Attributes = append(coinBase.Attributes, &txAttr) - transactions := []*core.Transaction{coinBase, elaCoin} + transactions := []*types.Transaction{coinBase, elaCoin} hashes := make([]common.Uint256, 0, len(transactions)) for _, tx := range transactions { hashes = append(hashes, tx.Hash()) diff --git a/interface/interface.go b/interface/interface.go index d3cad7a..a4e7797 100644 --- a/interface/interface.go +++ b/interface/interface.go @@ -4,8 +4,8 @@ import ( "github.com/elastos/Elastos.ELA.SPV/bloom" "github.com/elastos/Elastos.ELA.SPV/database" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/types" ) // SPV service config @@ -55,13 +55,13 @@ type SPVService interface { // To verify if a transaction is valid // This method is useful when receive a transaction from other peer - VerifyTransaction(bloom.MerkleProof, core.Transaction) error + VerifyTransaction(bloom.MerkleProof, types.Transaction) error // Send a transaction to the P2P network - SendTransaction(core.Transaction) error + SendTransaction(types.Transaction) error // GetTransaction query a transaction by it's hash. - GetTransaction(txId *common.Uint256) (*core.Transaction, error) + GetTransaction(txId *common.Uint256) (*types.Transaction, error) // GetTransactionIds query all transaction hashes on the given block height. GetTransactionIds(height uint32) ([]*common.Uint256, error) @@ -97,7 +97,7 @@ type TransactionListener interface { Address() string // Type() indicates which transaction type this listener are interested - Type() core.TransactionType + Type() types.TransactionType // Flags control the notification actions by the given flag Flags() uint64 @@ -105,7 +105,7 @@ type TransactionListener interface { // Notify() is the method to callback the received transaction // with the merkle tree proof to verify it, the notifyId is key of this // notify message and it must be submitted with the receipt together. - Notify(notifyId common.Uint256, proof bloom.MerkleProof, tx core.Transaction) + Notify(notifyId common.Uint256, proof bloom.MerkleProof, tx types.Transaction) } func NewSPVService(config *Config) (SPVService, error) { diff --git a/interface/interface_test.go b/interface/interface_test.go index 1e726c8..810a3a5 100644 --- a/interface/interface_test.go +++ b/interface/interface_test.go @@ -13,13 +13,13 @@ import ( "github.com/elastos/Elastos.ELA.SPV/sync" "github.com/elastos/Elastos.ELA.SPV/wallet/store" - "github.com/elastos/Elastos.ELA.Utility/common" "github.com/elastos/Elastos.ELA.Utility/elalog" - "github.com/elastos/Elastos.ELA.Utility/p2p/addrmgr" - "github.com/elastos/Elastos.ELA.Utility/p2p/connmgr" - "github.com/elastos/Elastos.ELA.Utility/p2p/server" "github.com/elastos/Elastos.ELA.Utility/signal" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/types" + "github.com/elastos/Elastos.ELA/p2p/addrmgr" + "github.com/elastos/Elastos.ELA/p2p/connmgr" + "github.com/elastos/Elastos.ELA/p2p/server" "github.com/stretchr/testify/assert" ) @@ -28,7 +28,7 @@ type TxListener struct { log elalog.Logger service SPVService address string - txType core.TransactionType + txType types.TransactionType flags uint64 } @@ -36,7 +36,7 @@ func (l *TxListener) Address() string { return l.address } -func (l *TxListener) Type() core.TransactionType { +func (l *TxListener) Type() types.TransactionType { return l.txType } @@ -44,7 +44,7 @@ func (l *TxListener) Flags() uint64 { return l.flags } -func (l *TxListener) Notify(id common.Uint256, proof bloom.MerkleProof, tx core.Transaction) { +func (l *TxListener) Notify(id common.Uint256, proof bloom.MerkleProof, tx types.Transaction) { l.log.Infof("Notify Type %s, TxID %s", tx.TxType.Name(), tx.Hash()) err := l.service.VerifyTransaction(proof, tx) if !assert.NoError(l.t, err) { @@ -77,14 +77,14 @@ func TestGetListenerKey(t *testing.T) { var key1, key2 common.Uint256 listener := &TxListener{ address: "ENTogr92671PKrMmtWo3RLiYXfBTXUe13Z", - txType: core.CoinBase, + txType: types.CoinBase, flags: FlagNotifyConfirmed | FlagNotifyInSyncing, } key1 = getListenerKey(listener) key2 = getListenerKey(&TxListener{ address: "ENTogr92671PKrMmtWo3RLiYXfBTXUe13Z", - txType: core.CoinBase, + txType: types.CoinBase, flags: FlagNotifyConfirmed | FlagNotifyInSyncing, }) if !key1.IsEqual(key2) { @@ -103,7 +103,7 @@ func TestGetListenerKey(t *testing.T) { // same address, flags different type key1 = getListenerKey(listener) - listener.txType = core.TransferAsset + listener.txType = types.TransferAsset key2 = getListenerKey(listener) if key1.IsEqual(key2) { t.Errorf("listeners with different type got same key %s", key1.String()) @@ -172,7 +172,7 @@ func TestNewSPVService(t *testing.T) { log: listlog, service: service, address: "8ZNizBf4KhhPjeJRGpox6rPcHE5Np6tFx3", - txType: core.CoinBase, + txType: types.CoinBase, flags: FlagNotifyConfirmed | FlagNotifyInSyncing, } @@ -181,7 +181,7 @@ func TestNewSPVService(t *testing.T) { log: listlog, service: service, address: "8ZNizBf4KhhPjeJRGpox6rPcHE5Np6tFx3", - txType: core.TransferAsset, + txType: types.TransferAsset, flags: 0, } diff --git a/interface/iutil/header.go b/interface/iutil/header.go index a11114e..1da15b1 100644 --- a/interface/iutil/header.go +++ b/interface/iutil/header.go @@ -3,15 +3,15 @@ package iutil import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/types" ) // Ensure Header implement BlockHeader interface. var _ util.BlockHeader = (*Header)(nil) type Header struct { - *core.Header + *types.Header } func (h *Header) Previous() common.Uint256 { @@ -30,6 +30,6 @@ func (h *Header) PowHash() common.Uint256 { return h.AuxPow.ParBlockHeader.Hash() } -func NewHeader(orgHeader *core.Header) util.BlockHeader { +func NewHeader(orgHeader *types.Header) util.BlockHeader { return &Header{Header: orgHeader} } diff --git a/interface/iutil/tx.go b/interface/iutil/tx.go index 668dace..945bfc0 100644 --- a/interface/iutil/tx.go +++ b/interface/iutil/tx.go @@ -3,13 +3,13 @@ package iutil import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/core/types" ) var _ util.Transaction = (*Tx)(nil) type Tx struct { - *core.Transaction + *types.Transaction } func (tx *Tx) MatchFilter(bf util.Filter) bool { @@ -46,6 +46,6 @@ func (tx *Tx) MatchFilter(bf util.Filter) bool { return false } -func NewTx(tx *core.Transaction) *Tx { +func NewTx(tx *types.Transaction) *Tx { return &Tx{tx} } diff --git a/interface/keystore.go b/interface/keystore.go index 3360187..56b1652 100644 --- a/interface/keystore.go +++ b/interface/keystore.go @@ -1,7 +1,7 @@ package _interface import ( - "github.com/elastos/Elastos.ELA.Utility/crypto" + "github.com/elastos/Elastos.ELA/crypto" ) /* diff --git a/interface/spvservice.go b/interface/spvservice.go index 7482915..ea607f8 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -15,9 +15,9 @@ import ( "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/types" + "github.com/elastos/Elastos.ELA/p2p/msg" ) const ( @@ -117,7 +117,7 @@ func (s *spvservice) SubmitTransactionReceipt(notifyId, txHash common.Uint256) e return s.db.Que().Del(¬ifyId, &txHash) } -func (s *spvservice) VerifyTransaction(proof bloom.MerkleProof, tx core.Transaction) error { +func (s *spvservice) VerifyTransaction(proof bloom.MerkleProof, tx types.Transaction) error { // Get Header from main chain header, err := s.headers.Get(&proof.BlockHash) if err != nil { @@ -154,17 +154,17 @@ func (s *spvservice) VerifyTransaction(proof bloom.MerkleProof, tx core.Transact return nil } -func (s *spvservice) SendTransaction(tx core.Transaction) error { +func (s *spvservice) SendTransaction(tx types.Transaction) error { return s.IService.SendTransaction(iutil.NewTx(&tx)) } -func (s *spvservice) GetTransaction(txId *common.Uint256) (*core.Transaction, error) { +func (s *spvservice) GetTransaction(txId *common.Uint256) (*types.Transaction, error) { utx, err := s.db.Txs().Get(txId) if err != nil { return nil, err } - var tx core.Transaction + var tx types.Transaction err = tx.Deserialize(bytes.NewReader(utx.RawData)) if err != nil { return nil, err @@ -386,7 +386,7 @@ func (s *spvservice) BlockCommitted(block *util.Block) { continue } - var tx core.Transaction + var tx types.Transaction err = tx.Deserialize(bytes.NewReader(utx.RawData)) if err != nil { continue @@ -429,7 +429,7 @@ func (s *spvservice) Close() error { } func (s *spvservice) queueMessageByListener( - listener TransactionListener, tx *core.Transaction, height uint32) { + listener TransactionListener, tx *types.Transaction, height uint32) { // skip unpacked transaction if height == 0 { return @@ -449,7 +449,7 @@ func (s *spvservice) queueMessageByListener( } func (s *spvservice) notifyTransaction(notifyId common.Uint256, - proof bloom.MerkleProof, tx core.Transaction, + proof bloom.MerkleProof, tx types.Transaction, confirmations uint32) (TransactionListener, bool) { listener, ok := s.listeners[notifyId] @@ -494,10 +494,10 @@ func getListenerKey(listener TransactionListener) common.Uint256 { return sha256.Sum256(buf.Bytes()) } -func getConfirmations(tx core.Transaction) uint32 { +func getConfirmations(tx types.Transaction) uint32 { // TODO user can set confirmations attribute in transaction, // if the confirmation attribute is set, use it instead of default value - if tx.TxType == core.CoinBase { + if tx.TxType == types.CoinBase { return 100 } return DefaultConfirmations diff --git a/interface/store/addrs.go b/interface/store/addrs.go index 8ca6029..c92bd97 100644 --- a/interface/store/addrs.go +++ b/interface/store/addrs.go @@ -5,7 +5,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/sdk" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/interface/store/databatch.go b/interface/store/databatch.go index 5786ae3..1a8059a 100644 --- a/interface/store/databatch.go +++ b/interface/store/databatch.go @@ -7,7 +7,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/core/types" "github.com/syndtr/goleveldb/leveldb" ) @@ -50,14 +50,14 @@ func (b *dataBatch) DelAll(height uint32) error { return err } - var tx core.Transaction + var tx types.Transaction err = tx.Deserialize(bytes.NewReader(utx.RawData)) if err != nil { return err } for index := range tx.Outputs { - outpoint := core.NewOutPoint(utx.Hash, uint16(index)) + outpoint := types.NewOutPoint(utx.Hash, uint16(index)) b.Batch.Delete(toKey(BKTOps, outpoint.Bytes()...)) } diff --git a/interface/store/headers.go b/interface/store/headers.go index 6799d2c..a6bdb22 100644 --- a/interface/store/headers.go +++ b/interface/store/headers.go @@ -11,7 +11,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" "github.com/cevaris/ordered_map" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/interface/store/interface.go b/interface/store/interface.go index dd24fc0..f841be8 100644 --- a/interface/store/interface.go +++ b/interface/store/interface.go @@ -7,7 +7,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) type HeaderStore interface { diff --git a/interface/store/ops.go b/interface/store/ops.go index 0103beb..e114973 100644 --- a/interface/store/ops.go +++ b/interface/store/ops.go @@ -5,7 +5,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/syndtr/goleveldb/leveldb" dbutil "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/interface/store/opsbatch.go b/interface/store/opsbatch.go index 28830c2..c8b4cf5 100644 --- a/interface/store/opsbatch.go +++ b/interface/store/opsbatch.go @@ -5,7 +5,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/interface/store/que.go b/interface/store/que.go index a898cf0..2b5b418 100644 --- a/interface/store/que.go +++ b/interface/store/que.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/interface/store/que_test.go b/interface/store/que_test.go index bcfa440..f45975f 100644 --- a/interface/store/que_test.go +++ b/interface/store/que_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/stretchr/testify/assert" "github.com/syndtr/goleveldb/leveldb" diff --git a/interface/store/quebatch.go b/interface/store/quebatch.go index ab46009..177f97b 100644 --- a/interface/store/quebatch.go +++ b/interface/store/quebatch.go @@ -4,7 +4,7 @@ import ( "encoding/binary" "sync" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" diff --git a/interface/store/txs.go b/interface/store/txs.go index 1ed73f3..e187f65 100644 --- a/interface/store/txs.go +++ b/interface/store/txs.go @@ -7,7 +7,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/syndtr/goleveldb/leveldb" dbutil "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/interface/store/txs_test.go b/interface/store/txs_test.go index bf9781c..bbe2ae3 100644 --- a/interface/store/txs_test.go +++ b/interface/store/txs_test.go @@ -7,7 +7,7 @@ import ( math "math/rand" "testing" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) func TestTxIds(t *testing.T) { diff --git a/interface/store/txsbatch.go b/interface/store/txsbatch.go index 50a0ca3..4555e0b 100644 --- a/interface/store/txsbatch.go +++ b/interface/store/txsbatch.go @@ -7,7 +7,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/log.go b/log.go index 8fbf1dd..f0d9f90 100644 --- a/log.go +++ b/log.go @@ -13,9 +13,9 @@ import ( "github.com/elastos/Elastos.ELA.Utility/elalog" "github.com/elastos/Elastos.ELA.Utility/http/jsonrpc" - "github.com/elastos/Elastos.ELA.Utility/p2p/addrmgr" - "github.com/elastos/Elastos.ELA.Utility/p2p/connmgr" - "github.com/elastos/Elastos.ELA.Utility/p2p/server" + "github.com/elastos/Elastos.ELA/p2p/addrmgr" + "github.com/elastos/Elastos.ELA/p2p/connmgr" + "github.com/elastos/Elastos.ELA/p2p/server" ) const LogPath = "./logs-spv/" diff --git a/peer/log.go b/peer/log.go index 1993995..3a1c0ae 100644 --- a/peer/log.go +++ b/peer/log.go @@ -2,7 +2,7 @@ package peer import ( "github.com/elastos/Elastos.ELA.Utility/elalog" - "github.com/elastos/Elastos.ELA.Utility/p2p/peer" + "github.com/elastos/Elastos.ELA/p2p/peer" ) // log is a logger that is initialized with no output filters. This diff --git a/peer/peer.go b/peer/peer.go index a50a1a8..a35596b 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -8,10 +8,10 @@ import ( "github.com/elastos/Elastos.ELA.SPV/bloom" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA.Utility/p2p/peer" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/p2p" + "github.com/elastos/Elastos.ELA/p2p/msg" + "github.com/elastos/Elastos.ELA/p2p/peer" ) const ( diff --git a/sdk/account.go b/sdk/account.go index 988375c..f4a2d47 100644 --- a/sdk/account.go +++ b/sdk/account.go @@ -1,8 +1,11 @@ package sdk import ( - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/crypto" + "bytes" + "errors" + + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/crypto" ) /* @@ -22,15 +25,12 @@ type Account struct { // Create an account instance with private key and public key func NewAccount(privateKey []byte, publicKey *crypto.PublicKey) (*Account, error) { - signatureRedeemScript, err := crypto.CreateStandardRedeemScript(publicKey) + redeemScript, err := createCheckSigRedeemScript(publicKey) if err != nil { return nil, err } - programHash, err := crypto.ToProgramHash(signatureRedeemScript) - if err != nil { - return nil, err - } + programHash := common.ToProgramHash(common.PrefixStandard, redeemScript) address, err := programHash.ToAddress() if err != nil { @@ -40,7 +40,7 @@ func NewAccount(privateKey []byte, publicKey *crypto.PublicKey) (*Account, error return &Account{ privateKey: privateKey, publicKey: publicKey, - redeemScript: signatureRedeemScript, + redeemScript: redeemScript, programHash: programHash, address: address, }, nil @@ -79,3 +79,16 @@ func (a *Account) Sign(data []byte) ([]byte, error) { } return signature, nil } + +func createCheckSigRedeemScript(publicKey *crypto.PublicKey) ([]byte, error) { + content, err := publicKey.EncodePoint(true) + if err != nil { + return nil, errors.New("create standard redeem script, encode public key failed") + } + buf := new(bytes.Buffer) + buf.WriteByte(byte(len(content))) + buf.Write(content) + buf.WriteByte(byte(common.STANDARD)) + + return buf.Bytes(), nil +} \ No newline at end of file diff --git a/sdk/addrfilter.go b/sdk/addrfilter.go index 16d8e79..e1129f9 100644 --- a/sdk/addrfilter.go +++ b/sdk/addrfilter.go @@ -3,7 +3,7 @@ package sdk import ( "sync" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) /* diff --git a/sdk/crypto.go b/sdk/crypto.go index c1e9486..f792609 100644 --- a/sdk/crypto.go +++ b/sdk/crypto.go @@ -5,7 +5,7 @@ import ( "crypto/elliptic" "crypto/rand" - . "github.com/elastos/Elastos.ELA.Utility/crypto" + . "github.com/elastos/Elastos.ELA/crypto" ) // Generate a ECC private key by the given curve diff --git a/sdk/interface.go b/sdk/interface.go index b249fc2..3d95733 100644 --- a/sdk/interface.go +++ b/sdk/interface.go @@ -4,7 +4,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) /* diff --git a/sdk/protocal.go b/sdk/protocal.go index 4536feb..070590c 100644 --- a/sdk/protocal.go +++ b/sdk/protocal.go @@ -1,7 +1,7 @@ package sdk import ( - "github.com/elastos/Elastos.ELA.Utility/p2p" + "github.com/elastos/Elastos.ELA/p2p" ) const ( diff --git a/sdk/service.go b/sdk/service.go index a3f8eef..a6f27b8 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -12,11 +12,11 @@ import ( "github.com/elastos/Elastos.ELA.SPV/sync" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" - "github.com/elastos/Elastos.ELA.Utility/p2p/peer" - "github.com/elastos/Elastos.ELA.Utility/p2p/server" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/p2p" + "github.com/elastos/Elastos.ELA/p2p/msg" + "github.com/elastos/Elastos.ELA/p2p/peer" + "github.com/elastos/Elastos.ELA/p2p/server" ) const ( diff --git a/spvwallet.go b/spvwallet.go index 7dd6d30..8d3e62f 100644 --- a/spvwallet.go +++ b/spvwallet.go @@ -13,11 +13,13 @@ import ( "github.com/elastos/Elastos.ELA.SPV/wallet/store/sqlite" "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/crypto" "github.com/elastos/Elastos.ELA.Utility/http/jsonrpc" httputil "github.com/elastos/Elastos.ELA.Utility/http/util" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/contract/program" + "github.com/elastos/Elastos.ELA/core/types" + "github.com/elastos/Elastos.ELA/core/types/payload" + "github.com/elastos/Elastos.ELA/crypto" ) const ( @@ -61,7 +63,7 @@ func (w *spvwallet) putTx(batch sqlite.DataBatch, utx util.Transaction, // Filter address if w.getAddrFilter().ContainAddr(output.ProgramHash) { var lockTime = output.OutputLock - if tx.TxType == core.CoinBase { + if tx.TxType == types.CoinBase { lockTime = height + 100 } utxo := sutil.NewUTXO(txId, height, index, output.Value, lockTime, output.ProgramHash) @@ -350,7 +352,7 @@ func NewWallet() (*spvwallet, error) { } func newTransaction() util.Transaction { - return sutil.NewTx(&core.Transaction{}) + return sutil.NewTx(&types.Transaction{}) } // GenesisHeader creates a specific genesis header by the given @@ -360,22 +362,22 @@ func GenesisHeader() util.BlockHeader { genesisTime := time.Date(2017, time.December, 22, 10, 0, 0, 0, time.UTC) // header - header := core.Header{ - Version: core.BlockVersion, + header := types.Header{ + Version: 0, Previous: common.EmptyHash, MerkleRoot: common.EmptyHash, Timestamp: uint32(genesisTime.Unix()), Bits: 0x1d03ffff, - Nonce: core.GenesisNonce, + Nonce: types.GenesisNonce, Height: uint32(0), } // ELA coin - elaCoin := &core.Transaction{ - TxType: core.RegisterAsset, + elaCoin := &types.Transaction{ + TxType: types.RegisterAsset, PayloadVersion: 0, - Payload: &core.PayloadRegisterAsset{ - Asset: core.Asset{ + Payload: &payload.PayloadRegisterAsset{ + Asset: payload.Asset{ Name: "ELA", Precision: 0x08, AssetType: 0x00, @@ -383,31 +385,31 @@ func GenesisHeader() util.BlockHeader { Amount: 0 * 100000000, Controller: common.Uint168{}, }, - Attributes: []*core.Attribute{}, - Inputs: []*core.Input{}, - Outputs: []*core.Output{}, - Programs: []*core.Program{}, + Attributes: []*types.Attribute{}, + Inputs: []*types.Input{}, + Outputs: []*types.Output{}, + Programs: []*program.Program{}, } - coinBase := &core.Transaction{ - TxType: core.CoinBase, - PayloadVersion: core.PayloadCoinBaseVersion, - Payload: new(core.PayloadCoinBase), - Inputs: []*core.Input{ + coinBase := &types.Transaction{ + TxType: types.CoinBase, + PayloadVersion: payload.PayloadCoinBaseVersion, + Payload: new(payload.PayloadCoinBase), + Inputs: []*types.Input{ { - Previous: core.OutPoint{ + Previous: types.OutPoint{ TxID: common.EmptyHash, Index: 0x0000, }, Sequence: 0x00000000, }, }, - Attributes: []*core.Attribute{}, + Attributes: []*types.Attribute{}, LockTime: 0, - Programs: []*core.Program{}, + Programs: []*program.Program{}, } - coinBase.Outputs = []*core.Output{ + coinBase.Outputs = []*types.Output{ { AssetID: elaCoin.Hash(), Value: 3300 * 10000 * 100000000, @@ -416,10 +418,10 @@ func GenesisHeader() util.BlockHeader { } nonce := []byte{0x4d, 0x65, 0x82, 0x21, 0x07, 0xfc, 0xfd, 0x52} - txAttr := core.NewAttribute(core.Nonce, nonce) + txAttr := types.NewAttribute(types.Nonce, nonce) coinBase.Attributes = append(coinBase.Attributes, &txAttr) - transactions := []*core.Transaction{coinBase, elaCoin} + transactions := []*types.Transaction{coinBase, elaCoin} hashes := make([]common.Uint256, 0, len(transactions)) for _, tx := range transactions { hashes = append(hashes, tx.Hash()) diff --git a/sync/manager.go b/sync/manager.go index e59e6e8..6c16d25 100644 --- a/sync/manager.go +++ b/sync/manager.go @@ -8,8 +8,8 @@ import ( "github.com/elastos/Elastos.ELA.SPV/peer" "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/p2p/msg" ) const ( diff --git a/util/header.go b/util/header.go index 22d3e25..140541b 100644 --- a/util/header.go +++ b/util/header.go @@ -5,8 +5,8 @@ import ( "fmt" "math/big" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/p2p/msg" ) // Header is a data structure stored in database. diff --git a/util/interface.go b/util/interface.go index 3eb4a38..4a3a66f 100644 --- a/util/interface.go +++ b/util/interface.go @@ -3,7 +3,7 @@ package util import ( "io" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) type BlockHeader interface { diff --git a/util/outpoint.go b/util/outpoint.go index c029638..f73ae07 100644 --- a/util/outpoint.go +++ b/util/outpoint.go @@ -4,7 +4,7 @@ import ( "bytes" "io" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) type OutPoint struct { diff --git a/util/tx.go b/util/tx.go index a021fca..e81e167 100644 --- a/util/tx.go +++ b/util/tx.go @@ -5,8 +5,8 @@ import ( "io" "time" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/p2p/msg" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/p2p/msg" ) // Tx is a data structure used in database. diff --git a/wallet/client.go b/wallet/client.go index 1ea459f..76a24bc 100644 --- a/wallet/client.go +++ b/wallet/client.go @@ -8,7 +8,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/wallet/client/transaction" "github.com/elastos/Elastos.ELA.SPV/wallet/client/wallet" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/urfave/cli" ) diff --git a/wallet/client/account/account.go b/wallet/client/account/account.go index 854ac1b..a943157 100644 --- a/wallet/client/account/account.go +++ b/wallet/client/account/account.go @@ -8,8 +8,8 @@ import ( "strings" "github.com/elastos/Elastos.ELA.SPV/wallet/client" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/crypto" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/crypto" "github.com/urfave/cli" ) diff --git a/wallet/client/common.go b/wallet/client/common.go index 9524c3a..41f2645 100644 --- a/wallet/client/common.go +++ b/wallet/client/common.go @@ -2,6 +2,7 @@ package client import ( "bufio" + "bytes" "errors" "fmt" "os" @@ -10,7 +11,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/howeyc/gopass" "github.com/urfave/cli" ) @@ -45,7 +46,7 @@ func GetPassword(password []byte, confirmed bool) ([]byte, error) { return nil, err } - if !common.IsEqualBytes(password, confirm) { + if !bytes.Equal(password, confirm) { return nil, errors.New("input password unmatched") } } diff --git a/wallet/client/database/database.go b/wallet/client/database/database.go index ea7e510..52ce62d 100644 --- a/wallet/client/database/database.go +++ b/wallet/client/database/database.go @@ -7,7 +7,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/wallet/store/sqlite" "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) func New() (*database, error) { diff --git a/wallet/client/database/interface.go b/wallet/client/database/interface.go index 48f6a00..9574bb0 100644 --- a/wallet/client/database/interface.go +++ b/wallet/client/database/interface.go @@ -3,7 +3,7 @@ package database import ( "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) type Database interface { diff --git a/wallet/client/keystore.go b/wallet/client/keystore.go index 7539090..0649ae1 100644 --- a/wallet/client/keystore.go +++ b/wallet/client/keystore.go @@ -1,6 +1,7 @@ package client import ( + "bytes" "crypto/rand" "crypto/sha256" "errors" @@ -8,8 +9,8 @@ import ( "sync" "github.com/elastos/Elastos.ELA.SPV/sdk" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/crypto" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/crypto" ) const ( @@ -152,10 +153,10 @@ func (store *Keystore) verifyPassword(password []byte) error { if err != nil { return err } - if common.IsEqualBytes(origin, passwordHash[:]) { - return nil + if !bytes.Equal(origin, passwordHash[:]) { + return errors.New("password wrong") } - return errors.New("password wrong") + return nil } func (store *Keystore) ChangePassword(oldPassword, newPassword []byte) error { diff --git a/wallet/client/keystore_file.go b/wallet/client/keystore_file.go index cd9d033..0e71ced 100644 --- a/wallet/client/keystore_file.go +++ b/wallet/client/keystore_file.go @@ -7,7 +7,7 @@ import ( "os" "sync" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) const ( @@ -28,8 +28,7 @@ type KeystoreFile struct { } func CreateKeystoreFile() (*KeystoreFile, error) { - - if common.FileExisted(KeystoreFilename) { + if info, err := os.Stat(KeystoreFilename); info != nil || os.IsExist(err) { return nil, errors.New("key store file already exist") } diff --git a/wallet/client/transaction/transaction.go b/wallet/client/transaction/transaction.go index 05ff3ee..4140e66 100644 --- a/wallet/client/transaction/transaction.go +++ b/wallet/client/transaction/transaction.go @@ -12,9 +12,9 @@ import ( "github.com/elastos/Elastos.ELA.SPV/wallet/client" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/crypto" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/types" + "github.com/elastos/Elastos.ELA/crypto" "github.com/urfave/cli" ) @@ -26,7 +26,7 @@ func CreateTransaction(c *cli.Context, wallet *client.Wallet) error { return output(txn) } -func createTransaction(c *cli.Context, wallet *client.Wallet) (*core.Transaction, error) { +func createTransaction(c *cli.Context, wallet *client.Wallet) (*types.Transaction, error) { feeStr := c.String("fee") if feeStr == "" { return nil, errors.New("use --fee to specify transfer fee") @@ -45,7 +45,7 @@ func createTransaction(c *cli.Context, wallet *client.Wallet) (*core.Transaction } } - var tx *core.Transaction + var tx *types.Transaction multiOutput := c.String("file") if multiOutput != "" { @@ -91,7 +91,7 @@ func createTransaction(c *cli.Context, wallet *client.Wallet) (*core.Transaction return tx, nil } -func createMultiOutputTransaction(c *cli.Context, wallet *client.Wallet, path, from string, fee *common.Fixed64) (*core.Transaction, error) { +func createMultiOutputTransaction(c *cli.Context, wallet *client.Wallet, path, from string, fee *common.Fixed64) (*types.Transaction, error) { if _, err := os.Stat(path); err != nil { return nil, errors.New("invalid multi output file path") } @@ -117,7 +117,7 @@ func createMultiOutputTransaction(c *cli.Context, wallet *client.Wallet, path, f } lockStr := c.String("lock") - var tx *core.Transaction + var tx *types.Transaction if lockStr == "" { tx, err = wallet.CreateMultiOutputTransaction(from, fee, multiOutput...) if err != nil { @@ -151,7 +151,7 @@ func SignTransaction(password []byte, context *cli.Context, wallet *client.Walle return output(txn) } -func signTransaction(password []byte, wallet *client.Wallet, tx *core.Transaction) (*core.Transaction, error) { +func signTransaction(password []byte, wallet *client.Wallet, tx *types.Transaction) (*types.Transaction, error) { haveSign, needSign, err := crypto.GetSignStatus(tx.Programs[0].Code, tx.Programs[0].Parameter) if haveSign == needSign { return nil, errors.New("transaction was fully signed, no need more sign") @@ -168,7 +168,7 @@ func signTransaction(password []byte, wallet *client.Wallet, tx *core.Transactio func SendTransaction(password []byte, context *cli.Context, wallet *client.Wallet) error { content, err := getContent(context) - var tx *core.Transaction + var tx *types.Transaction if content == nil { // Create transaction with command line arguments tx, err = createTransaction(context, wallet) @@ -181,7 +181,7 @@ func SendTransaction(password []byte, context *cli.Context, wallet *client.Walle return err } } else { - tx = new(core.Transaction) + tx = new(types.Transaction) data, err := common.HexStringToBytes(*content) if err != nil { return fmt.Errorf("Deseralize transaction file failed, error %s", err.Error()) @@ -234,7 +234,7 @@ func getContent(context *cli.Context) (*string, error) { return &content, nil } -func getTransaction(context *cli.Context) (*core.Transaction, error) { +func getTransaction(context *cli.Context) (*types.Transaction, error) { content, err := getContent(context) if err != nil { return nil, err @@ -245,7 +245,7 @@ func getTransaction(context *cli.Context) (*core.Transaction, error) { return nil, errors.New("decode transaction content failed") } - var txn core.Transaction + var txn types.Transaction err = txn.Deserialize(bytes.NewReader(rawData)) if err != nil { return nil, errors.New("deserialize transaction failed") @@ -254,7 +254,7 @@ func getTransaction(context *cli.Context) (*core.Transaction, error) { return &txn, nil } -func output(tx *core.Transaction) error { +func output(tx *types.Transaction) error { // Serialise transaction content buf := new(bytes.Buffer) tx.Serialize(buf) diff --git a/wallet/client/wallet.go b/wallet/client/wallet.go index 3f226f4..2f2a5cf 100644 --- a/wallet/client/wallet.go +++ b/wallet/client/wallet.go @@ -12,10 +12,12 @@ import ( "github.com/elastos/Elastos.ELA.SPV/wallet/client/database" "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA.Utility/crypto" "github.com/elastos/Elastos.ELA.Utility/http/jsonrpc" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/contract/program" + "github.com/elastos/Elastos.ELA/core/types" + "github.com/elastos/Elastos.ELA/core/types/payload" + "github.com/elastos/Elastos.ELA/crypto" ) var ( @@ -98,10 +100,7 @@ func (wallet *Wallet) AddMultiSignAccount(M uint, publicKeys ...*crypto.PublicKe return nil, errors.New("[Wallet], CreateStandardRedeemScript failed") } - programHash, err := crypto.ToProgramHash(redeemScript) - if err != nil { - return nil, errors.New("[Wallet], CreateMultiSignAddress failed") - } + programHash := common.ToProgramHash(common.PrefixMultisig, redeemScript) err = wallet.AddAddress(programHash, redeemScript, sutil.TypeMulti) if err != nil { @@ -115,28 +114,28 @@ func (wallet *Wallet) AddMultiSignAccount(M uint, publicKeys ...*crypto.PublicKe } func (wallet *Wallet) CreateTransaction(fromAddress, toAddress string, amount, - fee *common.Fixed64) (*core.Transaction, error) { + fee *common.Fixed64) (*types.Transaction, error) { return wallet.CreateLockedTransaction(fromAddress, toAddress, amount, fee, uint32(0)) } func (wallet *Wallet) CreateLockedTransaction(fromAddress, toAddress string, - amount, fee *common.Fixed64, lockedUntil uint32) (*core.Transaction, error) { + amount, fee *common.Fixed64, lockedUntil uint32) (*types.Transaction, error) { return wallet.CreateLockedMultiOutputTransaction(fromAddress, fee, lockedUntil, &Transfer{toAddress, amount}) } func (wallet *Wallet) CreateMultiOutputTransaction(fromAddress string, fee *common.Fixed64, - outputs ...*Transfer) (*core.Transaction, error) { + outputs ...*Transfer) (*types.Transaction, error) { return wallet.CreateLockedMultiOutputTransaction(fromAddress, fee, uint32(0), outputs...) } func (wallet *Wallet) CreateLockedMultiOutputTransaction(fromAddress string, fee *common.Fixed64, - lockedUntil uint32, outputs ...*Transfer) (*core.Transaction, error) { + lockedUntil uint32, outputs ...*Transfer) (*types.Transaction, error) { return wallet.createTransaction(fromAddress, fee, lockedUntil, outputs...) } func (wallet *Wallet) createTransaction(fromAddress string, fee *common.Fixed64, lockedUntil uint32, - outputs ...*Transfer) (*core.Transaction, error) { + outputs ...*Transfer) (*types.Transaction, error) { // Check if output is valid if outputs == nil || len(outputs) == 0 { return nil, errors.New("[Wallet], Invalid transaction target") @@ -149,7 +148,7 @@ func (wallet *Wallet) createTransaction(fromAddress string, fee *common.Fixed64, } // Create transaction outputs var totalOutputValue = common.Fixed64(0) // The total value will be spend - var txOutputs []*core.Output // The outputs in transaction + var txOutputs []*types.Output // The outputs in transaction totalOutputValue += *fee // Add transaction fee for _, output := range outputs { @@ -157,7 +156,7 @@ func (wallet *Wallet) createTransaction(fromAddress string, fee *common.Fixed64, if err != nil { return nil, errors.New("[Wallet], Invalid receiver address") } - txOutput := &core.Output{ + txOutput := &types.Output{ AssetID: sysAssetId, ProgramHash: *receiver, Value: *output.Value, @@ -175,7 +174,7 @@ func (wallet *Wallet) createTransaction(fromAddress string, fee *common.Fixed64, availableUTXOs = sutil.SortByValue(availableUTXOs) // Sort available UTXOs by value ASC // Create transaction inputs - var txInputs []*core.Input // The inputs in transaction + var txInputs []*types.Input // The inputs in transaction for _, utxo := range availableUTXOs { txInputs = append(txInputs, InputFromUTXO(utxo)) if utxo.Value < totalOutputValue { @@ -184,7 +183,7 @@ func (wallet *Wallet) createTransaction(fromAddress string, fee *common.Fixed64, totalOutputValue = 0 break } else if utxo.Value > totalOutputValue { - change := &core.Output{ + change := &types.Output{ AssetID: sysAssetId, Value: utxo.Value - totalOutputValue, OutputLock: uint32(0), @@ -207,7 +206,7 @@ func (wallet *Wallet) createTransaction(fromAddress string, fee *common.Fixed64, return wallet.newTransaction(addr.Script(), txInputs, txOutputs), nil } -func (wallet *Wallet) Sign(password []byte, tx *core.Transaction) (*core.Transaction, error) { +func (wallet *Wallet) Sign(password []byte, tx *types.Transaction) (*types.Transaction, error) { // Verify password err := wallet.VerifyPassword(password) if err != nil { @@ -239,10 +238,10 @@ func (wallet *Wallet) Sign(password []byte, tx *core.Transaction) (*core.Transac return tx, nil } -func (wallet *Wallet) signStandardTransaction(tx *core.Transaction) (*core.Transaction, error) { +func (wallet *Wallet) signStandardTransaction(tx *types.Transaction) (*types.Transaction, error) { code := tx.Programs[0].Code // Get signer - programHash, err := crypto.GetSigner(code) + programHash := common.ToProgramHash(common.PrefixStandard, code) // Check if current user is a valid signer account := wallet.Keystore.GetAccountByProgramHash(programHash) if account == nil { @@ -260,23 +259,24 @@ func (wallet *Wallet) signStandardTransaction(tx *core.Transaction) (*core.Trans buf.WriteByte(byte(len(signature))) buf.Write(signature) // Set program - var program = &core.Program{code, buf.Bytes()} - tx.Programs = []*core.Program{program} + tx.Programs = []*program.Program{{code, buf.Bytes()}} return tx, nil } -func (wallet *Wallet) signMultiSigTransaction(tx *core.Transaction) (*core.Transaction, error) { +func (wallet *Wallet) signMultiSigTransaction(tx *types.Transaction) (*types.Transaction, error) { code := tx.Programs[0].Code param := tx.Programs[0].Parameter // Check if current user is a valid signer var signerIndex = -1 - programHashes, err := crypto.GetSigners(code) + publicKeys, err := crypto.ParseMultisigScript(code) if err != nil { return nil, err } var account *sdk.Account - for i, programHash := range programHashes { + for i, publicKey := range publicKeys { + code := append(publicKey, common.STANDARD) + programHash := common.ToProgramHash(common.PrefixStandard, code) account = wallet.Keystore.GetAccountByProgramHash(programHash) if account != nil { signerIndex = i @@ -302,7 +302,7 @@ func (wallet *Wallet) signMultiSigTransaction(tx *core.Transaction) (*core.Trans return tx, nil } -func (wallet *Wallet) SendTransaction(tx *core.Transaction) error { +func (wallet *Wallet) SendTransaction(tx *types.Transaction) error { buf := new(bytes.Buffer) if err := tx.Serialize(buf); err != nil { return err @@ -331,31 +331,29 @@ func (wallet *Wallet) removeLockedUTXOs(utxos []*sutil.UTXO) []*sutil.UTXO { return availableUTXOs } -func InputFromUTXO(utxo *sutil.UTXO) *core.Input { - input := new(core.Input) +func InputFromUTXO(utxo *sutil.UTXO) *types.Input { + input := new(types.Input) input.Previous.TxID = utxo.Op.TxID input.Previous.Index = utxo.Op.Index input.Sequence = utxo.LockTime return input } -func (wallet *Wallet) newTransaction(redeemScript []byte, inputs []*core.Input, outputs []*core.Output) *core.Transaction { +func (wallet *Wallet) newTransaction(redeemScript []byte, inputs []*types.Input, outputs []*types.Output) *types.Transaction { // Create payload - txPayload := &core.PayloadTransferAsset{} + txPayload := &payload.PayloadTransferAsset{} // Create attributes - txAttr := core.NewAttribute(core.Nonce, []byte(strconv.FormatInt(rand.Int63(), 10))) - attributes := make([]*core.Attribute, 0) + txAttr := types.NewAttribute(types.Nonce, []byte(strconv.FormatInt(rand.Int63(), 10))) + attributes := make([]*types.Attribute, 0) attributes = append(attributes, &txAttr) - // Create program - var program = &core.Program{redeemScript, nil} // Create transaction - return &core.Transaction{ - TxType: core.TransferAsset, + return &types.Transaction{ + TxType: types.TransferAsset, Payload: txPayload, Attributes: attributes, Inputs: inputs, Outputs: outputs, - Programs: []*core.Program{program}, + Programs: []*program.Program{{redeemScript, nil}}, LockTime: wallet.BestHeight(), } } diff --git a/wallet/store/headers/cache.go b/wallet/store/headers/cache.go index 03e5ce0..1c58cd9 100644 --- a/wallet/store/headers/cache.go +++ b/wallet/store/headers/cache.go @@ -6,7 +6,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" "github.com/cevaris/ordered_map" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) type cache struct { diff --git a/wallet/store/headers/database.go b/wallet/store/headers/database.go index 77fbdae..d3b03d1 100644 --- a/wallet/store/headers/database.go +++ b/wallet/store/headers/database.go @@ -9,7 +9,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/wallet/store/sqlite/addrs.go b/wallet/store/sqlite/addrs.go index 4c8ddcc..c050924 100644 --- a/wallet/store/sqlite/addrs.go +++ b/wallet/store/sqlite/addrs.go @@ -6,7 +6,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) // Ensure addrs implement Addrs interface. diff --git a/wallet/store/sqlite/addrsbatch.go b/wallet/store/sqlite/addrsbatch.go index 9ad9ce5..9dfb83d 100644 --- a/wallet/store/sqlite/addrsbatch.go +++ b/wallet/store/sqlite/addrsbatch.go @@ -4,7 +4,7 @@ import ( "database/sql" "sync" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) // Ensure addrsBatch implement AddrsBatch interface. diff --git a/wallet/store/sqlite/interface.go b/wallet/store/sqlite/interface.go index 2667ac1..70e6ea0 100644 --- a/wallet/store/sqlite/interface.go +++ b/wallet/store/sqlite/interface.go @@ -4,7 +4,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) type DataStore interface { diff --git a/wallet/store/sqlite/stxos.go b/wallet/store/sqlite/stxos.go index b0a549c..7befb49 100644 --- a/wallet/store/sqlite/stxos.go +++ b/wallet/store/sqlite/stxos.go @@ -7,7 +7,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) const CreateSTXOsDB = `CREATE TABLE IF NOT EXISTS STXOs( diff --git a/wallet/store/sqlite/txs.go b/wallet/store/sqlite/txs.go index 474879f..a0b3cd2 100644 --- a/wallet/store/sqlite/txs.go +++ b/wallet/store/sqlite/txs.go @@ -9,7 +9,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) const CreateTxsDB = `CREATE TABLE IF NOT EXISTS Txs( diff --git a/wallet/store/sqlite/txsbatch.go b/wallet/store/sqlite/txsbatch.go index 7ea3a79..c317e55 100644 --- a/wallet/store/sqlite/txsbatch.go +++ b/wallet/store/sqlite/txsbatch.go @@ -6,7 +6,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) // Ensure txs implement Txs interface. diff --git a/wallet/store/sqlite/utxos.go b/wallet/store/sqlite/utxos.go index e3dedc3..ae36347 100644 --- a/wallet/store/sqlite/utxos.go +++ b/wallet/store/sqlite/utxos.go @@ -7,7 +7,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" "github.com/elastos/Elastos.ELA.SPV/wallet/sutil" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) const CreateUTXOsDB = `CREATE TABLE IF NOT EXISTS UTXOs( diff --git a/wallet/sutil/addr.go b/wallet/sutil/addr.go index a0645a5..30c15f5 100644 --- a/wallet/sutil/addr.go +++ b/wallet/sutil/addr.go @@ -1,6 +1,6 @@ package sutil -import "github.com/elastos/Elastos.ELA.Utility/common" +import "github.com/elastos/Elastos.ELA/common" const ( TypeMaster = 0 diff --git a/wallet/sutil/header.go b/wallet/sutil/header.go index 105447d..b63e2a3 100644 --- a/wallet/sutil/header.go +++ b/wallet/sutil/header.go @@ -3,15 +3,15 @@ package sutil import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/common" + "github.com/elastos/Elastos.ELA/core/types" ) // Ensure Header implement BlockHeader interface. var _ util.BlockHeader = (*Header)(nil) type Header struct { - *core.Header + *types.Header } func (h *Header) Previous() common.Uint256 { @@ -30,10 +30,10 @@ func (h *Header) PowHash() common.Uint256 { return h.AuxPow.ParBlockHeader.Hash() } -func NewHeader(orgHeader *core.Header) util.BlockHeader { +func NewHeader(orgHeader *types.Header) util.BlockHeader { return &Header{Header: orgHeader} } func NewEmptyHeader() util.BlockHeader { - return &Header{&core.Header{}} + return &Header{&types.Header{}} } diff --git a/wallet/sutil/stxo.go b/wallet/sutil/stxo.go index 1ad66fb..49bce2b 100644 --- a/wallet/sutil/stxo.go +++ b/wallet/sutil/stxo.go @@ -2,7 +2,7 @@ package sutil import ( "fmt" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) type STXO struct { diff --git a/wallet/sutil/tx.go b/wallet/sutil/tx.go index f699986..751c0f4 100644 --- a/wallet/sutil/tx.go +++ b/wallet/sutil/tx.go @@ -3,13 +3,13 @@ package sutil import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA/core" + "github.com/elastos/Elastos.ELA/core/types" ) var _ util.Transaction = (*Tx)(nil) type Tx struct { - *core.Transaction + *types.Transaction } func (tx *Tx) MatchFilter(bf util.Filter) bool { @@ -46,6 +46,6 @@ func (tx *Tx) MatchFilter(bf util.Filter) bool { return false } -func NewTx(tx *core.Transaction) *Tx { +func NewTx(tx *types.Transaction) *Tx { return &Tx{tx} } diff --git a/wallet/sutil/utxo.go b/wallet/sutil/utxo.go index 71cfd1b..d86439c 100644 --- a/wallet/sutil/utxo.go +++ b/wallet/sutil/utxo.go @@ -6,7 +6,7 @@ import ( "github.com/elastos/Elastos.ELA.SPV/util" - "github.com/elastos/Elastos.ELA.Utility/common" + "github.com/elastos/Elastos.ELA/common" ) type UTXO struct { From 00681007051429d937076f8bd80383008bfd4652 Mon Sep 17 00:00:00 2001 From: jiangzehua Date: Tue, 15 Jan 2019 17:26:06 +0800 Subject: [PATCH 69/73] modify glide.yaml --- glide.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glide.yaml b/glide.yaml index 123faf3..e6137b9 100644 --- a/glide.yaml +++ b/glide.yaml @@ -8,9 +8,9 @@ import: - package: github.com/syndtr/goleveldb/leveldb - package: github.com/cevaris/ordered_map - package: github.com/elastos/Elastos.ELA.Utility - version: release_v0.1.1 + version: release_v0.1.2 - package: github.com/elastos/Elastos.ELA - version: release_v0.2.1 + version: release_v0.2.2 subpackages: - bloom - core From 06820620a9168fd8d7c5b717a78da7b50fb74064 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Fri, 4 Jan 2019 15:01:17 +0800 Subject: [PATCH 70/73] move data folder to ./data --- client.go | 2 +- config.go | 2 ++ log.go | 5 ++--- main.go | 2 +- spvwallet.go | 6 +++--- wallet/client.go | 4 ++-- wallet/client/database/database.go | 16 +++++++++------- wallet/client/wallet.go | 8 +++++--- wallet/store/headers/database.go | 5 +++-- wallet/store/sqlite/database.go | 6 +++--- 10 files changed, 31 insertions(+), 25 deletions(-) diff --git a/client.go b/client.go index 12c5962..da91ff2 100644 --- a/client.go +++ b/client.go @@ -15,7 +15,7 @@ var Version string func main() { url := fmt.Sprint("http://127.0.0.1:", config.JsonRpcPort, "/spvwallet") - wallet.RunClient(Version, url, getSystemAssetId()) + wallet.RunClient(Version, dataDir, url, getSystemAssetId()) } func getSystemAssetId() common.Uint256 { diff --git a/config.go b/config.go index de3b00c..c2ca233 100644 --- a/config.go +++ b/config.go @@ -12,6 +12,8 @@ import ( ) const ( + dataDir = "./data" + ConfigFilename = "./config.json" ) diff --git a/log.go b/log.go index f0d9f90..d4dbddf 100644 --- a/log.go +++ b/log.go @@ -3,6 +3,7 @@ package main import ( "io" "os" + "path/filepath" "github.com/elastos/Elastos.ELA.SPV/blockchain" "github.com/elastos/Elastos.ELA.SPV/peer" @@ -18,14 +19,12 @@ import ( "github.com/elastos/Elastos.ELA/p2p/server" ) -const LogPath = "./logs-spv/" - // log is a logger that is initialized with no output filters. This // means the package will not perform any logging by default until the caller // requests it. var ( fileWriter = elalog.NewFileWriter( - LogPath, + filepath.Join(dataDir, "logs"), config.MaxPerLogSize, config.MaxLogsSize, ) diff --git a/main.go b/main.go index 319de2d..6c3fa05 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,7 @@ func main() { interrupt := signal.NewInterrupt() // Create the SPV wallet instance. - w, err := NewWallet() + w, err := NewWallet(dataDir) if err != nil { waltlog.Error("Initiate SPV service failed,", err) os.Exit(0) diff --git a/spvwallet.go b/spvwallet.go index 8d3e62f..88165cb 100644 --- a/spvwallet.go +++ b/spvwallet.go @@ -305,14 +305,14 @@ func (w *spvwallet) sendTransaction(params httputil.Params) (interface{}, error) return nil, w.SendTransaction(tx) } -func NewWallet() (*spvwallet, error) { +func NewWallet(dataDir string) (*spvwallet, error) { // Initialize headers db - headers, err := headers.NewDatabase() + headers, err := headers.NewDatabase(dataDir) if err != nil { return nil, err } - db, err := sqlite.NewDatabase() + db, err := sqlite.NewDatabase(dataDir) if err != nil { return nil, err } diff --git a/wallet/client.go b/wallet/client.go index 76a24bc..f58b075 100644 --- a/wallet/client.go +++ b/wallet/client.go @@ -12,8 +12,8 @@ import ( "github.com/urfave/cli" ) -func RunClient(version, rpcUrl string, assetId common.Uint256) { - client.Setup(rpcUrl, assetId) +func RunClient(version, dataDir, rpcUrl string, assetId common.Uint256) { + client.Setup(dataDir, rpcUrl, assetId) app := cli.NewApp() app.Name = "ELASTOS SPV WALLET" diff --git a/wallet/client/database/database.go b/wallet/client/database/database.go index 52ce62d..b7c2547 100644 --- a/wallet/client/database/database.go +++ b/wallet/client/database/database.go @@ -10,21 +10,23 @@ import ( "github.com/elastos/Elastos.ELA/common" ) -func New() (*database, error) { - dataStore, err := sqlite.NewDatabase() +func New(dataDir string) (*database, error) { + dataStore, err := sqlite.NewDatabase(dataDir) if err != nil { return nil, err } return &database{ - lock: new(sync.RWMutex), - store: dataStore, + lock: new(sync.RWMutex), + dataDir: dataDir, + store: dataStore, }, nil } type database struct { - lock *sync.RWMutex - store sqlite.DataStore + lock *sync.RWMutex + dataDir string + store sqlite.DataStore } func (d *database) AddAddress(address *common.Uint168, script []byte, addrType int) error { @@ -80,7 +82,7 @@ func (d *database) Clear() error { d.lock.Lock() defer d.lock.Unlock() - headers, err := headers.NewDatabase() + headers, err := headers.NewDatabase(d.dataDir) if err != nil { return err } diff --git a/wallet/client/wallet.go b/wallet/client/wallet.go index 2f2a5cf..5a67ee3 100644 --- a/wallet/client/wallet.go +++ b/wallet/client/wallet.go @@ -21,6 +21,7 @@ import ( ) var ( + dataPath string jsonRpcUrl string sysAssetId common.Uint256 ) @@ -35,7 +36,8 @@ type Wallet struct { *Keystore } -func Setup(rpcUrl string, assetId common.Uint256) { +func Setup(dataDir, rpcUrl string, assetId common.Uint256) { + dataPath = dataDir jsonRpcUrl = rpcUrl sysAssetId = assetId } @@ -46,7 +48,7 @@ func Create(password []byte) error { return err } - db, err := database.New() + db, err := database.New(dataPath) if err != nil { return err } @@ -57,7 +59,7 @@ func Create(password []byte) error { } func Open() (*Wallet, error) { - db, err := database.New() + db, err := database.New(dataPath) if err != nil { return nil, err } diff --git a/wallet/store/headers/database.go b/wallet/store/headers/database.go index d3b03d1..bd52003 100644 --- a/wallet/store/headers/database.go +++ b/wallet/store/headers/database.go @@ -3,6 +3,7 @@ package headers import ( "encoding/hex" "fmt" + "path/filepath" "sync" "github.com/elastos/Elastos.ELA.SPV/database" @@ -29,8 +30,8 @@ var ( BKTChainTip = []byte("B") ) -func NewDatabase() (*Database, error) { - db, err := leveldb.OpenFile("header", nil) +func NewDatabase(dataDir string) (*Database, error) { + db, err := leveldb.OpenFile(filepath.Join(dataDir, "header"), nil) if err != nil { return nil, err } diff --git a/wallet/store/sqlite/database.go b/wallet/store/sqlite/database.go index 1c59161..550d315 100644 --- a/wallet/store/sqlite/database.go +++ b/wallet/store/sqlite/database.go @@ -3,6 +3,7 @@ package sqlite import ( "database/sql" "fmt" + "path/filepath" "sync" _ "github.com/mattn/go-sqlite3" @@ -10,7 +11,6 @@ import ( const ( DriverName = "sqlite3" - DBName = "./spv_wallet.db" ) // Ensure database implement DataStore interface @@ -27,8 +27,8 @@ type database struct { stxos *stxos } -func NewDatabase() (*database, error) { - db, err := sql.Open(DriverName, DBName) +func NewDatabase(dataDir string) (*database, error) { + db, err := sql.Open(DriverName, filepath.Join(dataDir, "wallet.db")) if err != nil { fmt.Println("Open sqlite db error:", err) return nil, err From 309361c5391fb47392d1a1203dcec72c53a847d6 Mon Sep 17 00:00:00 2001 From: AlexPan Date: Tue, 22 Jan 2019 17:38:10 +0800 Subject: [PATCH 71/73] change sdk/service.go Config/GetFilterData() to GetFilter() --- interface/spvservice.go | 15 ++++++++------- sdk/interface.go | 14 +++----------- sdk/service.go | 15 +-------------- spvwallet.go | 20 +++++++++++++++++--- 4 files changed, 29 insertions(+), 35 deletions(-) diff --git a/interface/spvservice.go b/interface/spvservice.go index ea607f8..d49fddb 100644 --- a/interface/spvservice.go +++ b/interface/spvservice.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "errors" "fmt" + "math" "os" "time" @@ -87,7 +88,7 @@ func newSpvService(cfg *Config) (*spvservice, error) { ChainStore: chainStore, NewTransaction: newTransaction, NewBlockHeader: newBlockHeader, - GetFilterData: service.GetFilterData, + GetFilter: service.GetFilter, StateNotifier: service, } @@ -181,13 +182,13 @@ func (s *spvservice) HeaderStore() database.Headers { return s.headers } -func (s *spvservice) GetFilterData() ([]*common.Uint168, []*util.OutPoint) { - ops, err := s.db.Ops().GetAll() - if err != nil { - log.Error("[SPV_SERVICE] GetData error ", err) +func (s *spvservice) GetFilter() *bloom.Filter { + addrs := s.db.Addrs().GetAll() + filter := bloom.NewFilter(uint32(len(addrs)), math.MaxUint32, 0) + for _, address := range addrs { + filter.Add(address.Bytes()) } - - return s.db.Addrs().GetAll(), ops + return filter } func (s *spvservice) putTx(batch store.DataBatch, utx util.Transaction, diff --git a/sdk/interface.go b/sdk/interface.go index 3d95733..4b710a4 100644 --- a/sdk/interface.go +++ b/sdk/interface.go @@ -1,10 +1,9 @@ package sdk import ( + "github.com/elastos/Elastos.ELA.SPV/bloom" "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/util" - - "github.com/elastos/Elastos.ELA/common" ) /* @@ -83,15 +82,8 @@ type Config struct { // NewBlockHeader create a new block header instance. NewBlockHeader func() util.BlockHeader - // GetFilterData() returns two arguments. - // First arguments are all addresses stored in your data store. - // Second arguments are all balance references to those addresses stored in your data store, - // including UTXO(Unspent Transaction Output)s and STXO(Spent Transaction Output)s. - // Outpoint is a data structure include a transaction ID and output index. It indicates the - // reference of an transaction output. If an address ever received an transaction output, - // there will be the outpoint reference to it. Any time you want to spend the balance of an - // address, you must provide the reference of the balance which is an outpoint in the transaction input. - GetFilterData func() ([]*common.Uint168, []*util.OutPoint) + // GetFilter() returns a transaction filter like a bloom filter or others. + GetFilter func() *bloom.Filter // StateNotifier is an optional config, if you don't want to receive state changes of transactions // or blocks, just keep it blank. diff --git a/sdk/service.go b/sdk/service.go index a6f27b8..253e333 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -2,7 +2,6 @@ package sdk import ( "fmt" - "math/rand" "os" "time" @@ -136,19 +135,7 @@ func (s *service) start() { } func (s *service) updateFilter() *bloom.Filter { - addresses, outpoints := s.cfg.GetFilterData() - elements := uint32(len(addresses) + len(outpoints)) - - filter := bloom.NewFilter(elements, rand.Uint32(), 0) - for _, address := range addresses { - filter.Add(address.Bytes()) - } - - for _, op := range outpoints { - filter.Add(op.Bytes()) - } - - return filter + return s.cfg.GetFilter() } func (s *service) makeEmptyMessage(cmd string) (p2p.Message, error) { diff --git a/spvwallet.go b/spvwallet.go index 88165cb..e0db186 100644 --- a/spvwallet.go +++ b/spvwallet.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/elastos/Elastos.ELA.SPV/bloom" "github.com/elastos/Elastos.ELA.SPV/database" "github.com/elastos/Elastos.ELA.SPV/sdk" "github.com/elastos/Elastos.ELA.SPV/util" @@ -184,7 +185,7 @@ func (w *spvwallet) Close() error { return w.db.Close() } -func (w *spvwallet) GetFilterData() ([]*common.Uint168, []*util.OutPoint) { +func (w *spvwallet) GetFilter() *bloom.Filter { utxos, err := w.db.UTXOs().GetAll() if err != nil { waltlog.Debugf("GetAll UTXOs error: %v", err) @@ -201,7 +202,20 @@ func (w *spvwallet) GetFilterData() ([]*common.Uint168, []*util.OutPoint) { outpoints = append(outpoints, stxo.Op) } - return w.getAddrFilter().GetAddrs(), outpoints + addrs := w.getAddrFilter().GetAddrs() + + elements := uint32(len(addrs) + len(outpoints)) + + filter := bloom.NewFilter(elements, 0, 0) + for _, addr := range addrs { + filter.Add(addr.Bytes()) + } + + for _, op := range outpoints { + filter.Add(op.Bytes()) + } + + return filter } func (w *spvwallet) NotifyNewAddress(hash []byte) { @@ -333,7 +347,7 @@ func NewWallet(dataDir string) (*spvwallet, error) { ChainStore: chainStore, NewTransaction: newTransaction, NewBlockHeader: sutil.NewEmptyHeader, - GetFilterData: w.GetFilterData, + GetFilter: w.GetFilter, StateNotifier: &w, }) if err != nil { From 7ea6c08035d286a5771ccebe2c9f392a32019e6e Mon Sep 17 00:00:00 2001 From: AlexPan Date: Mon, 28 Jan 2019 12:12:46 +0800 Subject: [PATCH 72/73] do not disconnect peer for tx notfound message --- sdk/service.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/sdk/service.go b/sdk/service.go index 253e333..1436e07 100644 --- a/sdk/service.go +++ b/sdk/service.go @@ -405,11 +405,18 @@ func (s *service) onTx(sp *speer.Peer, msgTx util.Transaction) { } func (s *service) onNotFound(sp *speer.Peer, notFound *msg.NotFound) { - // Every thing we requested was came from this connected peer, so - // no reason it said I have some data you don't have and when you - // come to get it, it say oh I didn't have it. - log.Warnf("Peer %s is sending us notFound -- disconnecting", sp) - sp.Disconnect() + // Some times when we com to get a transaction, it has been cleared from + // peer's mempool, so we get this notfound message, in that case, we just + // ignore it. But if we come to get blocks and get this message, then the + // peer is misbehaving, we disconnect it. + for _, iv := range notFound.InvList { + if iv.Type == msg.InvTypeTx { + continue + } + + log.Warnf("Peer %s is sending us notFound -- disconnecting", sp) + sp.Disconnect() + } } func (s *service) onReject(sp *speer.Peer, reject *msg.Reject) { From 7cd30a6c478e5b461b49a1bd622dc00227b79ae7 Mon Sep 17 00:00:00 2001 From: BochengZhang Date: Tue, 19 Feb 2019 15:40:47 +0800 Subject: [PATCH 73/73] Update glide.yaml --- glide.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glide.yaml b/glide.yaml index e6137b9..04f0416 100644 --- a/glide.yaml +++ b/glide.yaml @@ -8,9 +8,9 @@ import: - package: github.com/syndtr/goleveldb/leveldb - package: github.com/cevaris/ordered_map - package: github.com/elastos/Elastos.ELA.Utility - version: release_v0.1.2 + version: v0.1.2 - package: github.com/elastos/Elastos.ELA - version: release_v0.2.2 + version: v0.2.2 subpackages: - bloom - core