Stand alone backend

README.md

@@ -1 +1,63 @@
-Any experimental code should go here before it makes its way into production in one of the other repositories. Here, you have loose restrictions and are free to push and collaborate on code. To keep things organized, **please work in your own branch**; in other words, the main branch should remain empty apart from this document. When you are done experimenting and are ready to commit the code to production, find the respective channel in Discussions (such as Mobile) and ask for feedback. Once at least 5 people explicitly state that your code is stable and useful, you may then create a PR to move it into the respective repository. While I can't enforce this 5-person rule, note that it is mandatory and there are consequences for not following the rules. Anyone can create a branch at any time to experiment, no approval is needed for that. Good luck!
+# Goals
+
+1. Create a stand alone backend that any frontend (gui, web, cli) should
+   be able to use directly. This should enable all fontend tools to access
+   the same general feature set and avoid chance of errors between tools.
+
+2. Remove the use of tmp files. A normal user does not know that there are
+   additional copies made, so if picocrypt were to crash at the wrong moment,
+   they would not know to go find and clean up the tmp files. Add data saved
+   to disk should exist only at the clear intended destination.
+
+3. Make the encryption testable. At a minimum, each feature should have a
+   full integration test, to prove that any changes to picocrypt still work.
+   Ideally, there would be backwards compatibility testing, but I am not sure
+   how to implement that yet.
+
+# Design
+
+Most of the design change I am proposing is motivated by 2 above - removing
+tmp files entirely. All of the current data processing picocrypt does can be
+done on each 1 MiB block at a time.
+
+## reed_solomon.go
+
+This file handles all of the reed solomon encryption/decryption work. In main
+picocrypt, it is up to the high level program logic to always use 128/136 byte
+blocks. In this refactor, I moved that responsibility to the RSEncoder and
+RSDecoder classes. The caller can pass any number of bytes to RSEncoder.Encode,
+and any extra bytes beyond the 128 byte chunks will be cached internally and
+used in the next call.
+
+The advantage of splitting the responsibility this way is that the higher level
+encoder does not need to know details about how rs works - it just passes data
+in, and gets data out. If we ever add changes later to how the rs encoding works
+(like making the amount of redundancy configurable), only the code here will
+need to change. The disadvantage is that there is more state to track, so it is
+more complex. There is also the chance that the input data does not exactly
+match 128 byte chunks, so a Flush call is necessary at the end to pad the last
+chunk. Looking for feedback on thoughts about this tradeoff.
+
+In main picocrypt, the files are decoded first by assuming the first 128 bytes
+are correct, and if the auth tag does not match at the end, then run the entire
+file again but force rs decoding. The reason for two passes is that decryption
+is very slow, and usually not needed, so assume the best and go fast. This 
+presents a problem for the no-tmp-file design, because I do not want to have to 
+make two passes over the data. An alternative option is to read in the first
+128 bytes and just re-encode them, seeing if that matches the current 136 bytes.
+If they match, the 128 bytes are almost definitely the original bytes. If they
+don't match, then run the expensive decoding and see if it is recoverable. This
+is slower than skipping entirely, but much faster than forcing decoding on every
+chunk.
+
+TODO: give brief overview of each source file here
+
+TODO: add tests
+
+TODO: add documentation
+
+TODO: add standard zip file support
+
+TODO: add recursive zip file support
+
+TODO: add file chunk support

encryption/ciphers.go

@@ -0,0 +1,95 @@
+package encryption
+
+import (
+	"crypto/cipher"
+	"github.com/HACKERALERT/serpent"
+	"golang.org/x/crypto/chacha20"
+	"golang.org/x/crypto/sha3"
+)
+
+func min(a, b int64) int64 {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+type EncryptionCipher struct {
+	paranoid     bool
+	chacha       *chacha20.Cipher
+	serpentBlock cipher.Block
+	serpent      cipher.Stream
+	keys         *Keys
+	counter      int64
+}
+
+func (ec *EncryptionCipher) Encode(dst, src []byte) {
+	i := int64(0)
+	for i < int64(len(src)) {
+		j := min(int64(len(src))-i, ResetNonceAt-ec.counter)
+		ec.chacha.XORKeyStream(dst[i:i+j], src[i:i+j])
+		if ec.paranoid {
+			ec.serpent.XORKeyStream(dst[i:i+j], dst[i:i+j])
+		}
+		ec.updateCounter(j)
+		i += j
+	}
+}
+
+func (ec *EncryptionCipher) updateCounter(length int64) {
+	ec.counter += length
+	if ec.counter < ResetNonceAt {
+		return
+	}
+	nonce := make([]byte, 24)
+	ec.keys.hkdf.Read(nonce)
+	ec.chacha, _ = chacha20.NewUnauthenticatedCipher(ec.keys.key, ec.keys.nonce)
+	serpentIV := make([]byte, 16)
+	ec.keys.hkdf.Read(serpentIV)
+	ec.serpent = cipher.NewCTR(ec.serpentBlock, serpentIV)
+	ec.counter = 0
+}
+
+func NewEncryptionCipher(keys *Keys, paranoid bool) *EncryptionCipher {
+	chacha, _ := chacha20.NewUnauthenticatedCipher(keys.key, keys.nonce)
+	sb, _ := serpent.NewCipher(keys.serpentKey)
+	s := cipher.NewCTR(sb, keys.serpentIV)
+	return &EncryptionCipher{paranoid, chacha, sb, s, keys, 0}
+}
+
+type Deniability struct {
+	key       []byte
+	salt      []byte
+	nonce     []byte
+	liveNonce []byte
+	chacha    *chacha20.Cipher
+	resetAt   int64
+	counter   int64
+}
+
+func (deny *Deniability) Deny(p []byte) {
+	i := int64(0)
+	for i < int64(len(p)) {
+		j := min(int64(len(p))-i, ResetNonceAt-deny.counter)
+		deny.chacha.XORKeyStream(p[i:i+j], p[i:i+j])
+		deny.updateCounter(j)
+		i += j
+	}
+}
+
+func (deny *Deniability) updateCounter(length int64) {
+	deny.counter += length
+	if deny.counter < ResetNonceAt {
+		return
+	}
+	tmp := sha3.New256()
+	tmp.Write(deny.liveNonce)
+	deny.liveNonce = tmp.Sum(nil)[:24]
+	deny.chacha, _ = chacha20.NewUnauthenticatedCipher(deny.key, deny.liveNonce)
+	deny.counter = 0
+}
+
+func NewDeniability(key, salt, nonce []byte) *Deniability {
+	chacha, _ := chacha20.NewUnauthenticatedCipher(key, nonce)
+	return &Deniability{key, salt, nonce, nonce, chacha, 60 * (1 << 20), 0}
+}

encryption/decrypt.go

@@ -0,0 +1,206 @@
+package encryption
+
+import (
+	"crypto/hmac"
+	"errors"
+	"fmt"
+	"golang.org/x/crypto/argon2"
+	"golang.org/x/crypto/blake2b"
+	"golang.org/x/crypto/sha3"
+	"hash"
+	"io"
+	"regexp"
+	"strconv"
+)
+
+type Decryptor struct {
+	password string
+	r        io.Reader
+	mac      hash.Hash
+	ec       *EncryptionCipher
+	deny     *Deniability
+	rs       *RSBodyDecoder
+	macTag   []byte
+
+	headerMask []byte
+	eof        bool
+	buffer     []byte
+	flushed    bool
+}
+
+func (d *Decryptor) Read(p []byte) (int, error) {
+	data := []byte{}
+	if !d.eof {
+		data = make([]byte, len(p))
+		n, err := d.r.Read(data)
+		data = data[:n]
+		if err == io.EOF {
+			d.eof = true
+		} else if err != nil {
+			return 0, err
+		}
+	}
+
+	var decodeErr error
+	if d.deny != nil {
+		d.deny.Deny(data)
+	}
+	if d.rs != nil && len(data) > 0 {
+		data, decodeErr = d.rs.Decode(data)
+	}
+	if d.eof && !d.flushed && d.rs != nil {
+		d.flushed = true
+		flushData, err := d.rs.Flush()
+		if errors.Is(err, ErrCorrupted) || decodeErr == nil {
+			decodeErr = err
+		}
+		data = append(data, flushData...)
+	}
+	d.mac.Write(data)
+	d.ec.Encode(data, data)
+	d.buffer = append(d.buffer, data...)
+
+	n := copy(p, d.buffer)
+	d.buffer = d.buffer[n:]
+	if (len(d.buffer) == 0) && d.eof {
+		macTag := d.mac.Sum(nil)
+		for i, m := range macTag {
+			if d.macTag[i] != m {
+				decodeErr = ErrCorrupted
+			}
+		}
+		if decodeErr == nil {
+			decodeErr = io.EOF
+		}
+	}
+	return n, decodeErr
+}
+
+func readFromHeader(r io.Reader, size int, deny *Deniability) ([]byte, error) {
+	if size == 0 {
+		return []byte{}, nil
+	}
+	tmp := make([]byte, size*3)
+	n, err := r.Read(tmp)
+	if (n != len(tmp)) || (err != nil) {
+		return []byte{}, err
+	}
+	if deny != nil {
+		deny.Deny(tmp)
+	}
+	data := make([]byte, size)
+	err = RSDecode(data, tmp)
+	if errors.Is(err, ErrCorrupted) {
+		return tmp, err
+	}
+	return data, err
+}
+
+var Version = "v1.99"
+
+func NewDecryptor(
+	pw string,
+	kf []io.Reader,
+	r io.Reader,
+) (*Decryptor, error) {
+
+	var deny *Deniability
+	headerDamaged := false
+
+	version, err := readFromHeader(r, 5, deny)
+	valid, _ := regexp.Match(`^v1\.\d{2}`, []byte(version))
+	if !valid {
+		data := make([]byte, 40)
+		copy(data, version[:15])
+		r.Read(data[15:])
+		key := argon2.IDKey([]byte(pw), data[:16], 4, 1<<20, 4, 32)
+		deny = NewDeniability(key, data[:16], data[16:])
+		fmt.Println(data)
+		version, err = readFromHeader(r, 5, deny)
+		valid, _ = regexp.Match(`^v1\.\d{2}`, version)
+		if !valid {
+			return nil, ErrCorrupted
+		}
+	}
+	if errors.Is(err, ErrRecoverable) {
+		headerDamaged = true
+	}
+
+	cLen, err := readFromHeader(r, 5, deny)
+	if errors.Is(err, ErrCorrupted) {
+		headerDamaged = true
+		fmt.Println("Bad")
+	} else {
+		c, _ := strconv.Atoi(string(cLen))
+		fmt.Print("Comment length: ")
+		fmt.Println(c)
+		for i := 0; i < c; i++ {
+			readFromHeader(r, 1, deny)
+		}
+	}
+
+	errs := make([]error, 10)
+	components := make([][]byte, 10)
+	components[2], errs[2] = readFromHeader(r, 5, deny)
+	components[3], errs[3] = readFromHeader(r, 16, deny)
+	components[4], errs[4] = readFromHeader(r, 32, deny)
+	components[5], errs[5] = readFromHeader(r, 16, deny)
+	components[6], errs[6] = readFromHeader(r, 24, deny)
+	components[7], errs[7] = readFromHeader(r, 64, deny)
+	components[8], errs[8] = readFromHeader(r, 32, deny)
+	components[9], errs[9] = readFromHeader(r, 64, deny)
+
+	for _, err := range errs {
+		if errors.Is(err, ErrRecoverable) {
+			headerDamaged = true
+		} else if err != nil {
+			return nil, err
+		}
+	}
+
+	paranoid := components[2][0] == 1
+	orderedKeyfiles := components[2][2] == 1
+
+	keys, err := NewKeys(
+		pw,
+		kf,
+		paranoid,
+		orderedKeyfiles,
+		components[3],
+		components[4],
+		components[5],
+		components[6],
+	)
+	// Generate keys. Allow duplicates in case the file was encrypted before
+	// catching duplicates was put in place.
+	if err != nil && !errors.Is(err, ErrDuplicateKeyfiles) {
+		return nil, err
+	}
+
+	var rs *RSBodyDecoder
+	if components[2][3] == 1 { // reed solomon bit is set
+		rs = &RSBodyDecoder{}
+	}
+
+	var mac hash.Hash
+	if components[2][0] == 1 { // paranoid
+		mac = hmac.New(sha3.New512, keys.macKey)
+	} else {
+		mac, _ = blake2b.New512(keys.macKey)
+	}
+
+	ec := NewEncryptionCipher(keys, paranoid)
+
+	decryptor := &Decryptor{
+		r:      r,
+		mac:    mac,
+		ec:     ec,
+		deny:   deny,
+		rs:     rs,
+		macTag: components[9],
+	}
+	if headerDamaged {
+		return decryptor, errors.New("Damaged header but recovered")
+	}
+	return decryptor, nil
+}