ads-service

#!/usr/bin/env python
from __future__ import print_function

import argparse
import getpass
import json
import hashlib
import os
import re
import shutil
import signal
import socket
import subprocess
import sys
import tarfile
import tempfile
import time
from glob import glob

try:
    from urllib2 import urlopen, Request, HTTPError
except ImportError:
    from urllib.request import urlopen
    from urllib.error import HTTPError

try:
    input = raw_input
except NameError:
    pass

DAEMON_BIN_NAME = 'adsd'
GENESIS_URL = 'https://raw.githubusercontent.com/adshares/ads/master/genesis.json'


def eprint(*args, **kwargs):
    """
    https://stackoverflow.com/a/14981125

    :param args: print args
    :param kwargs: print kwargs
    :return:
    """
    print(*args, file=sys.stderr, **kwargs)


def save_config(filepath, settings):
    """
    Save config in ADS format:

    key=value
    (one per line)

    A value can be a list (eg. [val1, val2]). This will be written like this:

    key=val1
    key=val2

    :param filepath: Config file path.
    :param settings: Dictionary of settings.
    :return:
    """
    with open(filepath, 'w') as f:
        for k, v in sorted(settings.items()):
            if isinstance(v, list):
                for seq_el in v:
                    f.write("{0}={1}\n".format(k, seq_el))
            else:
                f.write("{0}={1}\n".format(k, v))


class GenesisFile(object):
    """
    Genesis file object, with helper functions.
    """
    def __init__(self, genesis_file):
        """
        Read the file in.

        :param genesis_file: Genesis filepath or URL
        """

        if genesis_file.startswith('http'):
            try:
                genesis_response = urlopen(genesis_file)
                self.genesis_raw = genesis_response.read()
                self.genesis = json.loads(self.genesis_raw)
            except (HTTPError, ValueError) as e:
                self.genesis_raw = None
                self.genesis = {'nodes': []}

        self._genesis_dict()

    def _genesis_dict(self):
        self.nodes = {}
        for index, node in enumerate(self.genesis['nodes']):
            node_ident = '{0:04x}'.format(index + 1)
            self.nodes[node_ident] = node

    def node_identifier_from_public_key(self, pub_key):
        for node_id in self.nodes.keys():
            if self.nodes[node_id]['public_key'] == pub_key:
                return node_id

    def save(self, filepath):
        with open(filepath, 'w') as f:
            f.write(self.genesis_raw)


def validate_platform(verbose=False):
    """
    Check if you can run ADS. Currently supported only on 64 bit Linux.

    :return: Exits with status code 1 if platform is not valid, otherwise just prints out some system information.
    """

    # Platform check, linux or windows, or something else
    if not sys.platform.startswith('linux'):
        eprint("This platform is not supported for this configuration script.")
        eprint("I need 'linux', but this is '{0}'. See manual configuration instructions.".format(sys.platform))
        sys.exit(1)

    # Architecture check
    try:
        uname_data = os.uname()
        if uname_data[4] != 'x86_64':
            eprint("This architecture is not supported. I need *x86_64*.")
            sys.exit(1)
    except AttributeError:
        eprint("Can't detect the architecture. I need a *nix system to do that.")
        sys.exit(1)

    if verbose:
        print("Detected: {0} {1}".format(sys.platform, os.uname()[4]))


def get_my_ip(remote_ip="8.8.8.8", remote_port=53):  # NOSONAR
    """
    Try to connect to remote ip, to get local interface address. Defaults to google DNS.

    :param remote_ip: Remote ip to connect to.
    :param remote_port: Remote port to connect to.
    :return: Local interface ip.
    """

    # https://stackoverflow.com/a/166589

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect((remote_ip, remote_port))
    my_ip = s.getsockname()[0]
    s.close()

    return my_ip


def prepare_node_configuration(node_id, genesis_data, private_key, node_ident_genesis=None):

    if node_ident_genesis:
        print("Found matching public key in genesis.")
        # Check if private key matches public key in the chosen identifier
        if node_id and node_ident_genesis != node_id:
            eprint("Invalid private key for node: {0}".format(node_id))
            sys.exit(1)

        # Node id taken from matching public key
        node = genesis_data.nodes[node_ident_genesis]
        node['_nid'] = node_ident_genesis

        print("Configuring nodes: {0}".format(node_ident_genesis))
    else:
        print("No matching public key found in genesis file.")
        # New node (not in genesis)
        if not node_id:
            eprint("No node number provided to set up new node. Did you forget --node?")
            sys.exit(1)

        node = {'_nid': node_id,
                'accounts': []}
        print("Configuring new node: {0}".format(node_id))

    node['_secret'] = private_key
    return node


def configure(config, genesis_data):
    """
    Configure nodes and their first account

    :return:
    """
    node = {'_nid': config.node.upper(), '_secret': config.private_key, 'accounts': []}

    if not os.path.exists(config.working_dir):
        os.makedirs(config.working_dir)

    private_key_dir = os.path.join(config.working_dir, 'key')

    if not os.path.exists(private_key_dir):
        os.makedirs(private_key_dir)

    os.chmod(private_key_dir, 0o700)

    with open(os.path.join(private_key_dir, 'key.txt'), 'w') as f:
        f.write(node['_secret'])

    os.chmod(os.path.join(private_key_dir, 'key.txt'), 0o600)

    nconf = {'svid': int(node['_nid'], 16),
             'port': 6510,
             'offi': 6511,
             'addr': config.hostname}

    filepath = os.path.join(config.working_dir, 'options.cfg')

    save_config(filepath, nconf)
    print("{0} Saved options to: {1}".format(node['_nid'], filepath))

    print("Please remember to open ports 6510 and 6511.")


def get_public_key(private_key):
    proc = subprocess.Popen(['ads', '-w=/tmp', '-s'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output, errors = proc.communicate("{0}\n".format(private_key))
    match = re.search('Public key: (.*?)\s', errors)
    if match:
        return match.group(1)


def get_nodeid_from_genesis(genesis_data, private_key):
    pub_key = get_public_key(private_key)
    return genesis_data.node_identifier_from_public_key(pub_key)


def action_signal(working_dir, chosen_signal, name):
    """
    Send a signal to daemon

    :param working_dir: Path to directory holding the pid file.
    :param chosen_signal: Signal sent to daemon.
    :param name: Name of action.
    :return:
    """
    pid = get_daemon_pid(working_dir)
    if not pid:
        print("{0} not running in {1}.".format(DAEMON_BIN_NAME, working_dir))
    else:
        try:
            os.kill(pid, chosen_signal)
            print("ADS node {0} {1} was successful.".format(working_dir, name))
        except OSError:
            print("ADS node {0} {1} has failed.".format(working_dir, name))


def get_user_node_ident(genesis_data, private_key):
    node_ident_genesis = get_nodeid_from_genesis(genesis_data, private_key)
    node_ident_user = ''

    if node_ident_genesis:
        print("Public key for your secret key has been found in genesis file. Associated node identifier is {0}.".format(node_ident_genesis))
        while not re.match('^[0-9A-Fa-f]{4}$', node_ident_user):
            if node_ident_user:
                print("Invalid format. Node identifier is a 4 character hex value.")

            node_ident_user = input("Node identifier [{0}]: ".format(node_ident_genesis.upper()))
            if not node_ident_user:
                node_ident_user = node_ident_genesis

    else:
        print("Public key for your secret key has not been found in genesis file.")
        while not re.match('^[0-9A-Fa-f]{4}$', node_ident_user):
            if node_ident_user:
                print("Invalid format. Node identifier is a 4 character hex value.")
            node_ident_user = input("Node identifier: ")

    return node_ident_user.upper()


def validate_ip(ip_addr):
    pattern = '^(25[0-5]\.|2[0-4][0-9]\.|[01]?[0-9][0-9]?\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$'
    return re.match(pattern, ip_addr.strip())


def validate_private_key(private_key):
    pattern = '^[0-9a-fA-F]{64}$'
    return re.match(pattern, private_key.strip())

def rewrite_msid(msid_filepath, node_id):
    with open(msid_filepath, 'r+') as f:
        # Read
        old_msid = f.read()

        # Update msid
        new_msid = re.sub(r'^([A-F0-9]+) ([A-F0-9]+) ([A-F0-9X]+) ', r'\1 \2 {0} '.format(node_id), old_msid)

        # Overwrite
        f.seek(0)
        f.write(new_msid)
        f.truncate()

def read_backup_checksum(checksum_url):
    orig_hash = None
    req = Request(checksum_url, headers={'User-Agent': 'Mozilla/5.0'})
    response = urlopen(req)
    if response:
        orig_hash = response.read().split(' ')[0]

    return orig_hash

def get_backup(backup_url, output_path):

    print('Downloading DB, please wait.')
    # Download backup file
    req = Request(backup_url, headers={'User-Agent': 'Mozilla/5.0'})
    response = urlopen(req)

    with tempfile.TemporaryFile('r+b') as f:
        # Write the file
        while True:
            chunk = response.read(128 * 1024)
            if not chunk:
                break
            f.write(chunk)

        # Rewind and calculate hash
        f.seek(0)

        sha256 = hashlib.sha256()
        for b in iter(lambda: f.read(128 * 1024), b''):
            sha256.update(b)

        # Compare hashes
        download_hash = sha256.hexdigest()
        orig_hash = read_backup_checksum(backup_url + '.sha256')

        print('Original hash:      {0}'.format(orig_hash))
        print('File backup hash:   {0}'.format(download_hash))

        if orig_hash != download_hash:
            eprint('Backup validation failed. Please try again.')
            sys.exit(1)

        print('Backup validation successful.')
        # Extract backup
        print('Extracting backup.')
        f.seek(0)
        with tarfile.open(fileobj=f) as tf:
            tf.extractall(output_path)

def get_svid_from_options(options_filepath):

    svid = None

    with open(options_filepath) as f:
        line = f.readline()

        while not svid and line:
            match = re.match('^svid=(\d+)$', line)
            if match:
                svid = match.group(1)
            line = f.readline()

    return svid

def action_configure(conf_args):

    genesis_data = GenesisFile(GENESIS_URL)

    while True:
        private_key = getpass.getpass("Node's secret key: ")
        if validate_private_key(private_key):
            break
        else:
            print("Invalid format. Secret key is a 64 character hex value.")

    conf_args.private_key = private_key

    print("Default values are in []. Press enter to choose them.")
    conf_args.node = get_user_node_ident(genesis_data, conf_args.private_key)

    hostname = ''
    while not validate_ip(hostname):
        if hostname:
            print("Invalid format. Only IPv4 format is accepted.")
        hostname = input("Hostname [{0}]: ".format(get_my_ip()))
        if not hostname:
            hostname = get_my_ip()

    conf_args.hostname = hostname

    configure(conf_args, genesis_data)


def get_daemon_pid(working_dir=None):
    """
    Finds running adsd daemons.

    :param working_dir: Find specific daemon (optional)
    :return: Process identifier (pid) or None if no adsd daemons are running.
    """
    try:
        if working_dir:
            pid = subprocess.check_output(['pgrep', '-f', '{0}.*--work-dir={1}'.format(DAEMON_BIN_NAME, working_dir)])
        else:
            pid = subprocess.check_output(['pgrep', DAEMON_BIN_NAME])
            pid = pid.strip().split('\n')[0]  # Only one adsd should be running
        return int(pid)
    except subprocess.CalledProcessError:
        return None


def is_node_in_genesis(genesis_path, private_key):

    genesis_data = GenesisFile(genesis_path)
    node_ident_genesis = get_nodeid_from_genesis(genesis_data, private_key)
    return node_ident_genesis


def database_exists(working_dir):
    return glob(os.path.join(working_dir, 'usr', '*.dat'))


def action_start(working_dir, fast_sync):
    """
    :param working_dir: Path to node configuration directory.
    :return:
    """

    if not os.path.exists(working_dir):
        print("Working directory {0} does not exist.".format(working_dir))
        sys.exit(1)

    cmd = [
        DAEMON_BIN_NAME,
        '--work-dir={0}'.format(working_dir)
    ]

    if not database_exists(working_dir):

        if fast_sync:
            cmd += ['-f', '1']
        else:
            svid_int = int(get_svid_from_options(os.path.join(working_dir, 'options.cfg')))
            svid_hex = '{:0>4X}'.format(svid_int)
            get_backup('https://db.adshares.net/ads_node_db.tar.gz',
                       os.path.join(working_dir))

            rewrite_msid(os.path.join(working_dir, 'msid.txt'), svid_hex)

    stdout = open(os.path.join(working_dir, '{0}.log'.format(DAEMON_BIN_NAME)), 'a')
    stderr = open(os.path.join(working_dir, 'error.log'), 'a')

    proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)

    try:
        os.kill(proc.pid, 0)
        print("Process started: ", time.strftime("%Z - %Y/%m/%d, %H:%M:%S", time.localtime(time.time())))
    except OSError:
        eprint("Server not started.")
        sys.exit(1)

    wait_seconds = 3
    time.sleep(wait_seconds)

    if get_daemon_pid():
        print("ADS node {0} started.".format(working_dir))
    else:
        print("ADS node {0} failed to start in {1}.".format(working_dir, wait_seconds))
        sys.exit(1)


def prepare_args():

    if 'USER' in os.environ.keys():
        # Support for sudo
        default_working_dir = os.path.join(os.path.expanduser("~" + os.environ["USER"]), ".adsd")
    else:
        default_working_dir = os.path.expanduser("~/.adsd")

    parser = argparse.ArgumentParser(description='Maintenance service for the ADS network.')
    parser.add_argument('action', choices=['start', 'stop', 'restart', 'configure', 'status'])
    parser.add_argument('-w', '--working-dir', default=default_working_dir, help='working directory (default "{0}")'.format(default_working_dir))
    parser.add_argument('-f', '--force', action='store_true', help='Force killing a process when restarting a server.')
    parser.add_argument('-s', '--fast', action='store_true', help='Use a fast sync when joining the network.')

    args = parser.parse_args()

    args.bin_path = sys.argv[0]

    return args


def handle_start_action(args, daemon_running):

    if daemon_running:
        if args.force:
            action_signal(args.working_dir, signal.SIGKILL, 'forced stop')
        else:
            eprint("adsd already started. To force kill and start, use `{0} -f start`.".format(args.bin_path))
            sys.exit(1)

    action_start(args.working_dir, args.fast)


def handle_stop_action(args, daemon_running):

    if not daemon_running:
        print("adsd not running. To start, use `{0} start`.".format(args.bin_path))
        sys.exit(0)
    if args.force:
        action_signal(args.working_dir, signal.SIGKILL, 'forced stop')
    else:
        action_signal(args.working_dir, signal.SIGTERM, 'stop')


def handle_restart_action(args, daemon_running):

    if not daemon_running:
        action_start(args.working_dir)

    action_signal(args.working_dir, signal.SIGUSR1, 'restart')


def handle_configure_action(args, daemon_running):

    if daemon_running:
        eprint("adsd is running. Please stop it first, using `{0} stop` or `{0} --force stop`.".format(args.bin_path))
        sys.exit(1)

    if os.path.exists(args.working_dir):
        if args.force:
             shutil.rmtree(args.working_dir)
             os.mkdir(args.working_dir)
             eprint("Previous ADS node configuration removed.")
        else:
            eprint("ADS node configuration already exists; to overwrite it please use `{0} --force configure`.".format(args.bin_path))
            sys.exit(1)

    action_configure(args)


def decide_action(args):

    daemon_running = get_daemon_pid()

    if args.action == 'start':
        handle_start_action(args, daemon_running)

    elif args.action == 'stop':
        handle_stop_action(args, daemon_running)

    elif args.action == 'restart':
        handle_restart_action(args, daemon_running)

    elif args.action == 'configure':
        handle_configure_action(args, daemon_running)

    elif args.action == 'status':

        if daemon_running:
            print("adsd is running.")
        else:
            print("adsd is not running.")


if __name__ == '__main__':

    prepped_args = prepare_args()
    validate_platform()
    decide_action(prepped_args)