shell_connector.bash

#!/bin/bash

# Globals

# Store a unique boundary string generated by the 'mkBound' function.
# This boundary string is used to separate the output of different commands.
declare bound

# Store a unique boundary string specific to SQL responses.
# This boundary string is used to separate the output of different SQL commands.
declare sqlreqbound

# File descriptor associated with the input to the SQL client co-process.
# It is used to send SQL commands to the SQL client.
declare SQLIN

# File descriptor associated with the output from the SQL client co-process.
# It is used to read the responses of SQL commands from the SQL client.
declare SQLOUT

# File descriptor associated with the error stream of the SQL client co-process.
# It is used to read any error messages that the SQL client might produce.
declare SQLERR

# Store the last line read by 'newSqlConnector'.
# For debugging purposes.
declare lastsqlread


mkBound() {
    # mkBound() - Generate a unique boundary string
    # 
    # This function generates a unique boundary string that is used to delimit the output 
    # from sub-commands executed as co-processes, especially useful for SQL responses with variable length.
    # 
    # Parameters:
    # $1 (optional) - The name of the variable that will hold the generated boundary string. 
    #                 If not provided, defaults to a variable named "bound".
    # 
    # The generated boundary string is a sequence of 30 random alphanumeric characters and has the format: 
    # "--xxxxxxx-xxxx-xxxx-xxxx-xxxx-xxxx-", where "x" stands for a random character.
    # 
    # Constants:
    #   BOUNDARY_LENGTH - The length of the boundary string to be generated.
    #   BOUNDARY_ITERATIONS - The number of iterations in the loop for generating the boundary string.
    #   BOUNDARY_MASK - The mask used for selecting bits from the pseudo-random numbers.
    #   BOUNDARY_SHIFT - The number of bits to shift for the next character in the boundary string.
    # 
    # Notes: 
    # - Builds some uniq 30 random char string from 12 $RANDOM values:
    #   RANDOM is 15 bits. 15 x 2 = 30 bits -> 5 x 6 bits char
    # - The function uses bash's $RANDOM to generate pseudo-random numbers. This may not be suitable for 
    #   applications that require cryptographically strong random numbers.

    # Constants for boundary construction
    local BOUNDARY_LENGTH=30
    local BOUNDARY_ITERATIONS=6
    local BOUNDARY_MASK=63
    local BOUNDARY_SHIFT=6

    # Named reference to output variable
    local -n result=${1:-bound}

    # The string used for generating the boundary
    local _Bash64_refstr _out='' _l _i _num
    printf -v _Bash64_refstr "%s" {0..9} {a..z} {A..Z} @ _ 0

    # Generate boundary by looping BOUNDARY_ITERATIONS times
    # Construct BOUNDARY_LENGTH chars by shifting and masking RANDOM values
    # This is done by generating a pseudo-random number (_num) and then selecting a character from _Bash64_refstr
    # based on the bits in _num. The bits are selected by shifting _num to the right by _i bits and then applying
    # the BOUNDARY_MASK to select the least significant 6 bits. This is repeated BOUNDARY_LENGTH times to generate
    # the boundary string.
    for ((_l=BOUNDARY_ITERATIONS;_num=(RANDOM<<15|RANDOM),_l--;));do
	for ((_i=0;_i<BOUNDARY_LENGTH;_i+=BOUNDARY_SHIFT));do
	    _out+=${_Bash64_refstr:(_num>>_i)&BOUNDARY_MASK:1}
	done
    done

    # Format the output into "--xxxxxxx-xxxx-xxxx-xxxx-xxxx-xxxx-"
    printf -v result -- "--%s-%s-%s-%s-%s-%s-" "${_out:0:7}" \
	   "${_out:7:4}" "${_out:11:4}" "${_out:15:4}" \
	   "${_out:19:4}" "${_out:23}";
}

newConnector() {
    # newConnector() - Initiate a long-running subprocess for a given command and associates it with two file descriptors.
    #
    # This function sets up a long-running co-process using the provided command and arguments. It creates two
    # file descriptors XXIN and XXOUT associated with the co-process to manage data interaction, where 'XX' is the 
    # uppercase form of the command name. It also dynamically creates a function, named 'myXxx', to interact with the co-process.
    # 'Xxx' corresponds to the capitalized form of the command name.
    #
    # Synopsis:
    #   newConnector "command" "command_args" "check_input" "check_output" ["timeout"]
    #   newConnector "/path/to/command" "-arg1 -arg2" "initialization input" "expected response" [5]
    #
    # Parameters:
    #   command - The command to be run as a co-process.
    #   args - The arguments to be passed to the command.
    #   readiness_input - The initial input to be sent to the co-process for readiness verification.
    #   readiness_output - The expected output from the command when 'readiness_input' is sent as input.
    #   timeout - (Optional) The number of seconds to wait for a response from the co-process. Defaults to 3 seconds.
    #
    #
    # Returns:
    #   None directly. However, it prints a warning message to STDERR if the verification of the co-process fails.
    #
    # Constants:
    #   cinfd - File descriptor for input to the co-process (XXIN).
    #   coutfd - File descriptor for output from the co-process (XXOUT).
    #
    # Notes: 
    #   - The function 'myXxx' sends input to the command and reads the output into the 'result' variable.
    #   - The function 'myXxx' prints the result to STDOUT if there is no second argument.
    #   - If sending the 'readiness_input' does not yield the 'readiness_output' string, a warning is printed to STDERR.
    local command="$1" cmd=${1##*/} args="$2" readiness_input="$3" readiness_output="$4" timeout="${5:-3}"
    shift 5
    local initfile input
    local -n cinfd=${cmd^^}IN coutfd=${cmd^^}OUT

    # Start a new co-process using the provided command and arguments
    # The output of the command is unbuffered (-o0 option to stdbuf)
    # This co-process can interact with the main process via its standard input and output
    coproc stdbuf -o0 "$command" "$args" 2>&1
    # shellcheck disable=SC2034,SC2128
    cinfd=${COPROC[1]} coutfd=$COPROC 

    # Feed the initialization file to the command
    for initfile ;do
	cat >&"${cinfd}" "$initfile"
    done

    # Dynamically create a function that sends input to the command and reads its output
    # The function name is 'my' followed by the capitalized command name (e.g., myBc, myDate)
    # The function takes one argument, sends it to the command, and reads the response into the 'result' variable
    # shellcheck disable=SC1091
    source /dev/stdin <<-EOF
	my${cmd^}() {
	    local -n result=\${2:-${cmd}Out} # Name reference to the output variable
	    # Send input to the command
	    echo >&\${${cmd^^}IN} "\$1" &&
	    # Read the response with a timeout of $timeout seconds
	    read -u \${${cmd^^}OUT} -t "$timeout" result
	    # If there is no second argument, print the result
	    ((\$#>1)) || echo \$result
	}
	EOF

    # Check the command by sending the 'readiness_input' and comparing the response to 'readiness_output'
    my"${cmd^}" "$readiness_input" input
    if [ "$input" != "$readiness_output" ]; then
	printf >&2 "WARNING: Don't match! '%s' <> '%s'.\n" "$readiness_output" "$input"
    fi
}

newSqlConnector() {
    # newSqlConnector() - Establish a connection to a specified SQL client
    # 
    # Establishes a long-running connection to an SQL client, 
    # and prepares the environment necessary for executing SQL queries on that client.
    # It supports SQLite, MySQL, MariaDB, and PostgreSQL.
    # The output length is not fixed and could be empty.
    # 
    # Synopsis:
    #   newSqlConnector /usr/bin/sqlite3 $'-separator \t -header /dev/shm/test.sqlite'
    #   newSqlConnector /usr/bin/psql $'-Anh hostname -F \t --pset=footer=off user'
    #   newSqlConnector /usr/bin/mysql  '-h hostname -B -p database'
    #
    # Parameters:
    #   $1 (command) - The command to execute (i.e., the SQL client)
    #   $2 (args) - The command-line arguments for the SQL client
    #   $3 (readiness_input) and $4 (readiness_output) - Not used in the current function scope
    # 
    # Returns:
    #   None directly. But it sets up SQL input and output file descriptors 
    #   for subsequent interaction with the SQL client.
    #
    # Constants:
    #   COPROC - Array variable from the coproc keyword in bash, holding the file descriptors for co-process.
    #   SQLERR - A file descriptor for the SQL client's error stream
    # 
    # Notes: 
    # - The function assumes that the command passed to it is an executable, 
    #   so it should be validated before calling this function.
    # - If an unknown SQL client is passed, the function will print a warning but will not terminate.

    # Take the SQL client command and command arguments as input
    local command="$1" cmd=${1##*/} args readiness_input="$3" readiness_output="$4" COPROC
    # Split the command arguments
    IFS=' ' read -r -a args <<<"$2"
    # Create file descriptors for SQL input and output
    local -n _sqlin=SQLIN _sqlout=SQLOUT
    # Generate a unique boundary string
    mkBound bound
    # Determine the SQL client and set `sqlreqbound` for each client
    case $cmd in
	psql ) sqlreqbound='EXTRACT(\047EPOCH\047 FROM now())' ;;
	mysql|mariadb ) sqlreqbound='UNIX_TIMESTAMP()' ;;
	sqlite* ) sqlreqbound='STRFTIME(\047%%s\047,DATETIME(\047now\047))' ;;
	* ) 
        # If the SQL client is not recognized, give a warning
        echo >&2 "WARNING '$cmd' not known as SQL client";;
    esac

    # Create a new file descriptor for SQL error stream
    exec {SQLERR}<> <(: p)
    # Start the SQL client as a co-process, with its standard error redirected to `SQLERR`
    coproc stdbuf -o0 "$command" "${args[@]}" 2>&$SQLERR
    # Store the file descriptors for the co-process's standard input and output
    # shellcheck disable=SC2128
    _sqlin=${COPROC[1]} _sqlout=$COPROC 
}

mySqlReq() {
    # mySqlReq() - Send SQL commands to the co-process SQL client and manages responses
    #
    # This function takes the name of a variable as the first argument. It is designed to send an SQL command
    # to an SQL client, read the response, and store this response into the variable specified by the first argument. 
    # The function doesn't return any value but populates three variables based on the argument: `$1`, `${1}_h`, and `${1}_e`.
    # The variable `$1` contains the SQL response, `${1}_h` stores the header fields, and `${1}_e` holds any errors, if they occur.
    #
    # The SQL command to be executed can be passed in two ways:
    # 1. Directly as the second argument in the function call (e.g., mySqlReq result "SELECT * FROM table_name").
    # 2. Piped into the function via standard input if no second argument is provided (e.g., echo "SELECT * FROM table_name" | mySqlReq result).
    # When only the variable name is provided as an argument, the function waits for the SQL command from standard input.
    # In both cases, the result of the SQL command execution is stored in the variables as described.
    #
    # Synopsis:
    #   mySqlReq result "SELECT * FROM table_name"
    #   echo "SELECT * FROM table_name" | mySqlReq result
    #
    # Parameters:
    #   $1 - Name of the variable where the result of the SQL command will be stored.
    #   $@ - SQL command to be executed.
    #
    # Returns:
    #   Does not return a value but populates `$1`, `${1}_h`, and `${1}_e` variables.
    #
    # Constants:
    #   sqlreqbound - A unique boundary string that separates command outputs from SQL outputs.
    #
    # Notes: 
    # - The function heavily depends on the `newSqlConnector` function to create the SQL client co-process.
    # - It can handle responses from different SQL clients like sqlite, mysql, mariadb, and postgresql.

    # Initialize three variables for storing results, headers and any error messages.
    local -n result=$1 result_h=${1}_h result_e=${1}_e
    result=() result_h='' result_e=()
    local line
    # Set a timeout for reading from the SQL client co-process (in seconds).
    local timeout=0.02
    shift

    # If SQL command is provided as argument, send it to the SQL client co-process.
    # Otherwise, read SQL command from standard input and send it to the SQL client co-process.
    if (($#)) ;then
        printf >&"$SQLIN" '%s;\n' "${*%;}"
    else
        local -a _req
        mapfile -t _req
        printf >&"$SQLIN" '%s;\n' "${_req[*]%;}"
    fi

    # Request the output of the unique boundary string which was defined in the `newSqlConnector` function.
    printf >&"$SQLIN" 'SELECT '"${sqlreqbound}"' AS "%s";\n' "$bound"

    # Read the response from the SQL client co-process.
    local isFirstLine=true
    while read -ru "$SQLOUT" -t "$timeout" line; do
        # If the current line is equal to the boundary, break the loop
        if [ "$line" = "$bound" ]; then
            break
        fi
        if $isFirstLine; then
            # If the current line is the first line, store it in result_h and set isFirstLine to false
        # shellcheck disable=SC2034
            IFS=$'\t' read -r -a result_h <<< "$line"
            isFirstLine=false
        else
            # If the current line is not the first line, append it to the result array
            result+=("$line")
        fi
        done
    # shellcheck disable=SC2034
    lastsqlread="$line"

    # If there are any error messages available on the SQLERR file descriptor, read them into the `result_e` variable.
    while read -ru "$SQLERR" -t "$timeout" line; do
            result_e+=("$line")
        done
}