Fix and extend map server (#48)

Various Improvements and fixes for the map server:

* fix MHT root value bug (only allow single root value at each point in time)
* fix SPCT signature calculation (now includes timestamp in signature calculation)
* allow fetching certificates and policies with a single API call
* add option to insert single policy from file
* add helper option to verify policy chain constraints (used by browser extension's web assembly part)
* add documentation

README.md

@@ -1,6 +1,21 @@
 # FPKI
 
-TODO: This information is old and outdated.
+## Setting up local mapserver for testing
+
+- generate test policies:
+  `cd path/to/policy-generator; go run .`
+- clean DB:
+  `./tools/create_schema.sh`
+- ingest certificates (all `.csv` and `.gz` files located within a `bundled` subfolder are ingested):
+  `go run ./cmd/ingest path/to/cert-directory`
+- ingest root policy
+  `go run cmd/mapserver/main.go -policyFile path/to/policy-generator/output/pca.pc config.json`
+- ingest policies one by one
+  `for x in path/to/policy-generator/output/pc_*.pc; do go run cmd/mapserver/main.go -policyFile $x config.json; done`
+- run map server
+  `go run cmd/mapserver/main.go config.json`
+
+TODO: The information below is old and outdated.
 
 ## Features
 

cmd/ingest/certProcessor.go

@@ -202,7 +202,7 @@ func (p *CertificateProcessor) ConsolidateDB() {
 		if _, err := p.conn.DB().Exec(str); err != nil {
 			panic(fmt.Errorf("reenabling keys: %s", err))
 		}
-		str = "ALTER TABLE certs_aux_tmp ADD PRIMARY KEY (id)"
+		str = "ALTER TABLE certs_aux_tmp ADD PRIMARY KEY (cert_id)"
 		if _, err := p.conn.DB().Exec(str); err != nil {
 			panic(fmt.Errorf("reenabling keys: %s", err))
 		}

cmd/ingest/main.go

@@ -106,7 +106,6 @@ func mainFunction() int {
 	config := db.NewConfig(
 		mysql.WithDefaults(),
 		mysql.WithEnvironment(),
-		mysql.WithLocalSocket("/var/run/mysqld/mysqld.sock"),
 	)
 	conn, err := mysql.Connect(config)
 	exitIfError(err)

cmd/mapserver/main.go

@@ -1,17 +1,20 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"flag"
 	"fmt"
 	"os"
 	"syscall"
 	"time"
 
+	"github.com/netsec-ethz/fpki/pkg/common"
 	"github.com/netsec-ethz/fpki/pkg/db"
 	"github.com/netsec-ethz/fpki/pkg/db/mysql"
 	"github.com/netsec-ethz/fpki/pkg/mapserver"
 	"github.com/netsec-ethz/fpki/pkg/mapserver/config"
+	"github.com/netsec-ethz/fpki/pkg/mapserver/updater"
 	"github.com/netsec-ethz/fpki/pkg/util"
 )
 
@@ -31,9 +34,12 @@ func mainFunc() int {
 		fmt.Fprintf(os.Stderr, "Usage:\n%s configuration_file\n", os.Args[0])
 		flag.PrintDefaults()
 	}
-	updateVar := flag.Bool("updateNow", true, "Immediately trigger an update cycle")
+	flag.CommandLine.Usage = flag.Usage
+
+	updateVar := flag.Bool("updateNow", false, "Immediately trigger an update cycle")
 	createSampleConfig := flag.Bool("createSampleConfig", false,
 		"Create configuration file specified by positional argument")
+	insertPolicyVar := flag.String("policyFile", "", "policy certificate file to be ingested into the mapserver")
 	flag.Parse()
 
 	// We need the configuration file as the first positional argument.
@@ -43,9 +49,12 @@ func mainFunc() int {
 	}
 
 	var err error
-	if *createSampleConfig {
+	switch {
+	case *createSampleConfig:
 		err = writeSampleConfig()
-	} else {
+	case *insertPolicyVar != "":
+		err = insertPolicyFromFile(*insertPolicyVar)
+	default:
 		err = run(*updateVar)
 	}
 
@@ -54,6 +63,61 @@ func mainFunc() int {
 	return manageError(err)
 }
 
+func insertPolicyFromFile(policyFile string) error {
+	fmt.Printf("inserting policy from %s\n", policyFile)
+
+	ctx := context.Background()
+	// Load configuration and insert policy with it.
+	conf, err := config.ReadConfigFromFile(flag.Arg(0))
+	if err != nil {
+		return err
+	}
+	server, err := mapserver.NewMapServer(ctx, conf)
+	if err != nil {
+		return err
+	}
+	root, err := server.Conn.LoadRoot(ctx)
+	if err != nil {
+		return err
+	}
+
+	pc, err := util.PolicyCertificateFromFile(policyFile)
+	if err != nil {
+		return err
+	}
+
+	err = updater.UpdateWithKeepExisting(ctx, server.Conn, nil, nil, nil, nil, nil, []common.PolicyDocument{pc})
+	if err != nil {
+		return err
+	}
+
+	if err := server.Updater.CoalescePayloadsForDirtyDomains(ctx); err != nil {
+		return fmt.Errorf("coalescing payloads: %w", err)
+	}
+
+	// Update SMT.
+	if err := server.Updater.UpdateSMT(ctx); err != nil {
+		return fmt.Errorf("updating SMT: %w", err)
+	}
+
+	// Cleanup.
+	if err := server.Updater.Conn.CleanupDirty(ctx); err != nil {
+		return fmt.Errorf("cleaning up DB: %w", err)
+	}
+
+	newRoot, err := server.Conn.LoadRoot(ctx)
+	if err != nil {
+		return err
+	}
+	if bytes.Equal(root[:], newRoot[:]) {
+		fmt.Printf("MHT root value was not updated (%v)\n", newRoot)
+	} else {
+		fmt.Printf("MHT root value updated from %v to %v\n", root, newRoot)
+	}
+
+	return nil
+}
+
 func writeSampleConfig() error {
 	dbConfig := db.NewConfig(
 		mysql.WithDefaults(),
@@ -102,6 +166,11 @@ func runWithConfig(
 	if err != nil {
 		return err
 	}
+	root, err := server.Conn.LoadRoot(ctx)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Running map server with root: %v\n", root)
 
 	// Should update now?
 	if updateNow {
@@ -121,7 +190,7 @@ func runWithConfig(
 		})
 
 	// Listen in responder.
-	err = server.Listen(ctx)
+	err = server.ListenWithoutTLS(ctx)
 
 	// Regardless of the error, clean everything up.
 	cleanUp()

pkg/common/crypto/crypto.go

@@ -8,11 +8,42 @@ import (
 	"crypto/subtle"
 	"encoding/hex"
 	"fmt"
+	"regexp"
+	"strings"
+	"time"
 
 	"github.com/netsec-ethz/fpki/pkg/common"
 	"github.com/netsec-ethz/fpki/pkg/util"
 )
 
+func SignPolicyCertificateTimestamp(
+	pc *common.PolicyCertificate,
+	version int,
+	logId []byte,
+	key *rsa.PrivateKey,
+) (*common.SignedPolicyCertificateTimestamp, error) {
+
+	serializedPc, err := common.ToJSON(pc)
+	if err != nil {
+		return nil, fmt.Errorf("SignSPT | SerializePC | %w", err)
+	}
+
+	spt := common.NewSignedPolicyCertificateTimestamp(version, logId, time.Now(), nil)
+	signatureInput := common.NewSignedEntryTimestampSignatureInput(serializedPc, spt)
+	serializedSpt, err := common.ToJSON(signatureInput)
+	if err != nil {
+		return nil, fmt.Errorf("SignSPT | SerializeSPTInput | %w", err)
+	}
+
+	signature, err := SignBytes(serializedSpt, key)
+	if err != nil {
+		return nil, fmt.Errorf("SignSPT | Sign | %w", err)
+	}
+
+	spt.Signature = signature
+	return spt, nil
+}
+
 func SignBytes(b []byte, key *rsa.PrivateKey) ([]byte, error) {
 	hashOutput := sha256.Sum256(b)
 	signature, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, hashOutput[:])
@@ -181,6 +212,45 @@ func SignPolicyCertificateAsIssuer(
 	return nil
 }
 
+// VerifyIssuerConstraints verifies various constraints given by the issuer policy certificate
+// and returns the first violated constraint, or nil if no constraints were violated.
+func VerifyIssuerConstraints(
+	issuerPolCert *common.PolicyCertificate,
+	childPolCert *common.PolicyCertificate,
+) error {
+
+	// check validity period
+	if childPolCert.NotBefore.Before(issuerPolCert.NotBefore) {
+		return fmt.Errorf("policy certificate is valid before its issuer certificate (%v < %v)", childPolCert.NotBefore, issuerPolCert.NotBefore)
+	}
+	if issuerPolCert.NotAfter.Before(childPolCert.NotAfter) {
+		return fmt.Errorf("policy certificate is valid after its issuer certificate (%v > %v)", childPolCert.NotAfter, issuerPolCert.NotAfter)
+	}
+
+	// check domain constraint
+	// TODO (cyrill): could also check with public suffix list
+	validDomainName := regexp.MustCompile("([^.]+\\.)*([^.]+\\.?)?")
+	if !validDomainName.Match([]byte(childPolCert.DomainField)) {
+		return fmt.Errorf("Policy Certificate does not have a valid domain name: %s", childPolCert.DomainField)
+	}
+	if !validDomainName.Match([]byte(issuerPolCert.DomainField)) {
+		return fmt.Errorf("Issuer Policy Certificate does not have a valid domain name: %s", issuerPolCert.DomainField)
+	}
+	if issuerPolCert.DomainField != "" {
+		// all domain fields are accepted
+	} else if issuerPolCert.DomainField == childPolCert.DomainField {
+		// identical domain fields are accepted
+	} else {
+		if !strings.HasSuffix(childPolCert.DomainField, "."+issuerPolCert.DomainField) {
+			return fmt.Errorf("Policy certificate is not a subdomain of the issuer policy certificate")
+		} else {
+			// is valid subdomain
+		}
+	}
+
+	return nil
+}
+
 // VerifyIssuerSignature: used by domain owner, check whether CA signature is correct
 func VerifyIssuerSignature(
 	issuerPolCert *common.PolicyCertificate,

pkg/common/embedded_policies.go

@@ -26,11 +26,19 @@ type SignedPolicyCertificateTimestamp struct {
 	SignedEntryTimestamp
 }
 
+func (o SignedPolicyCertificateTimestamp) Raw() ([]byte, error) {
+	return rawTemplate(o)
+}
+
 // SignedPolicyCertificateRevocationTimestamp is a signed policy certificate revocation timestamp.
 type SignedPolicyCertificateRevocationTimestamp struct {
 	SignedEntryTimestamp
 }
 
+func (o SignedPolicyCertificateRevocationTimestamp) Raw() ([]byte, error) {
+	return rawTemplate(o)
+}
+
 func NewSignedEntryTimestamp(
 	version int,
 	logID []byte,
@@ -57,6 +65,32 @@ func (s SignedEntryTimestamp) Equal(x SignedEntryTimestamp) bool {
 		bytes.Equal(s.Signature, x.Signature)
 }
 
+type SignedEntryTimestampSignatureInput struct {
+	SignedEntryTimestamp
+	Entry []byte
+}
+
+func NewSignedEntryTimestampSignatureInput(
+	entry []byte,
+	spct *SignedPolicyCertificateTimestamp,
+) *SignedEntryTimestampSignatureInput {
+
+	return &SignedEntryTimestampSignatureInput{
+		SignedEntryTimestamp: *NewSignedEntryTimestamp(
+			spct.Version,
+			spct.LogID,
+			spct.AddedTS,
+			spct.Signature,
+		),
+		Entry: entry,
+	}
+}
+
+func (t SignedEntryTimestampSignatureInput) Equal(x SignedEntryTimestampSignatureInput) bool {
+	return t.SignedEntryTimestamp.Equal(x.SignedEntryTimestamp) &&
+		bytes.Equal(t.Entry, x.Entry)
+}
+
 func NewSignedPolicyCertificateTimestamp(
 	version int,
 	logID []byte,

pkg/common/json.go

@@ -77,6 +77,8 @@ func (*serializableObjectBase) marshalJSON(obj any) (string, []byte, error) {
 		T = "trillian.Proof"
 	case trilliantypes.LogRootV1:
 		T = "logrootv1"
+	case SignedEntryTimestampSignatureInput:
+		T = "spctsiginput"
 	default:
 		valOf := reflect.ValueOf(obj)
 		switch valOf.Kind() {
@@ -207,6 +209,8 @@ func (o *serializableObjectBase) unmarshalTypeObject(T string, data []byte) (boo
 		obj, err = inflateObj[trillian.Proof](data)
 	case "logrootv1":
 		obj, err = inflateObj[trilliantypes.LogRootV1](data)
+	case "spctsiginput":
+		obj, err = inflateObj[SignedEntryTimestampSignatureInput](data)
 	default:
 		err = fmt.Errorf("unknown type represented by \"%s\"", T)
 		obj = nil

pkg/common/json_test.go

@@ -170,3 +170,65 @@ func TestPolicyObjectBaseRaw(t *testing.T) {
 		})
 	}
 }
+
+// TestPolicyObjectBaseRaw checks that the Raw field of the PolicyObjectBase for any PolicyObject
+// that is rebuilt using our functions contains the original JSON.
+func TestPolicyObjectsRawImplementation(t *testing.T) {
+	testCases := map[string]struct {
+		obj any // Thing to serialize and deserialize and check Raw.
+		cmp func(obj1 any, obj2 any) bool
+	}{
+		"pc": {
+			obj: random.RandomPolicyCertificate(t),
+			cmp: func(obj1 any, obj2 any) bool {
+				obj2T := obj2.(*common.PolicyCertificate)
+				return obj1.(*common.PolicyCertificate).Equal(*obj2T)
+			},
+		},
+		"spct": {
+			obj: random.RandomSignedPolicyCertificateTimestamp(t),
+			cmp: func(obj1 any, obj2 any) bool {
+				obj2T := obj2.(*common.SignedPolicyCertificateTimestamp)
+				return obj1.(*common.SignedPolicyCertificateTimestamp).Equal(*obj2T)
+			},
+		},
+		"pcr": {
+			obj: random.RandomPolicyCertificateRevocation(t),
+			cmp: func(obj1 any, obj2 any) bool {
+				obj2T := obj2.(*common.PolicyCertificateRevocation)
+				return obj1.(*common.PolicyCertificateRevocation).Equal(*obj2T)
+			},
+		},
+		"spcrt": {
+			obj: random.RandomSignedPolicyCertificateRevocationTimestamp(t),
+			cmp: func(obj1 any, obj2 any) bool {
+				obj2T := obj2.(*common.SignedPolicyCertificateRevocationTimestamp)
+				return obj1.(*common.SignedPolicyCertificateRevocationTimestamp).Equal(*obj2T)
+			},
+		},
+		"pcsr": {
+			obj: random.RandomPolCertSignRequest(t),
+			cmp: func(obj1 any, obj2 any) bool {
+				// no comparison operator
+				return true
+				// return obj1.(common.PolicyCertificateSigningRequest).Equal(obj2.(common.PolicyCertificateSigningRequest))
+			},
+		},
+	}
+	for name, tc := range testCases {
+		name, tc := name, tc
+		t.Run(name, func(t *testing.T) {
+			// t.Parallel()
+			raw, err := tc.obj.(common.MarshallableDocument).Raw()
+			require.NoError(t, err)
+			// Serialize.
+			data, err := common.ToJSON(tc.obj)
+			require.NoError(t, err)
+			require.Equal(t, raw, data)
+			// Deserialize.
+			obj, err := common.FromJSON(data)
+			require.NoError(t, err)
+			require.True(t, tc.cmp(tc.obj, obj))
+		})
+	}
+}