add support for rules download in cli mode

Dockerfile

@@ -52,10 +52,7 @@ RUN cd /root && wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.3.2
 
 WORKDIR /home/deepfence/src/YaraHunter
 COPY . .
-RUN make clean \
-    && make all \
-    && cd /home/deepfence \
-    && git clone https://github.com/deepfence/yara-rules
+RUN make clean && make all
 
 
 FROM debian:bookworm
@@ -95,7 +92,6 @@ EOF
 RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y libgpgme-dev libdevmapper-dev
 
 WORKDIR /home/deepfence/usr
-COPY --from=builder /home/deepfence/yara-rules .
 COPY --from=builder /usr/local/yara.tar.gz /usr/local/yara.tar.gz
 COPY --from=builder /home/deepfence/src/YaraHunter/YaraHunter .
 COPY --from=builder /home/deepfence/src/YaraHunter/config.yaml .

Makefile

@@ -14,7 +14,7 @@ vendor: go.mod
 	go mod vendor
 
 yarahunter: vendor $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/**/*.go
-	CGO_LDFLAGS="-ljansson -lcrypto -lmagic" PKG_CONFIG_PATH=/usr/local/yara/lib/pkgconfig:$(PKG_CONFIG_PATH) go build -buildmode=pie -ldflags="-s -w -extldflags=-static" -buildvcs=false -v .
+	CGO_LDFLAGS="-ljansson -lcrypto -lmagic" PKG_CONFIG_PATH=/usr/local/yara/lib/pkgconfig:$(PKG_CONFIG_PATH) go build -buildmode=pie -ldflags="-s -w -extldflags=-static -X 'main.version=$(DF_IMG_TAG)'" -buildvcs=false -v .
 
 .PHONY: clean bootstrap
 

README.md

@@ -49,17 +49,37 @@ Pull the image that needs to be scanned for example `metal3d/xmrig` and scan it:
 
 ```
 docker pull metal3d/xmrig
+```
+
+Set Product and Licence and scan it:
 
+```
 docker run -i --rm --name=deepfence-yarahunter \
+     -e DEEPFENCE_PRODUCT=<ThreatMapper or ThreatStryker> \
+     -e DEEPFENCE_LICENSE=<ThreatMapper or ThreatStryker license key> \
      -v /var/run/docker.sock:/var/run/docker.sock \
      -v /tmp:/home/deepfence/output \
-     quay.io/deepfenceio/deepfence_malware_scanner_ce:2.3.0 \
+     quay.io/deepfenceio/deepfence_malware_scanner_ce:3.0.0 \
      --image-name metal3d/xmrig:latest \
      --output=json > xmrig-scan.json
 ```
 
 This returns, among other things, clear indication of the presence of XMRig. Note that we store the output (`xmrig-scan.json`) for quick and easy manipulation:
 
+Rules can also be cached to use next run by mounting a seperate path and passing `rules-path` argument
+```
+docker run -i --rm --name=deepfence-yarahunter \
+     -e DEEPFENCE_PRODUCT=<ThreatMapper or ThreatStryker> \
+     -e DEEPFENCE_LICENSE=<ThreatMapper or ThreatStryker license key> \
+     -v /var/run/docker.sock:/var/run/docker.sock \
+     -v /tmp:/home/deepfence/output \
+     -v /tmp/rules:/tmp/rules \
+     quay.io/deepfenceio/deepfence_malware_scanner_ce:3.0.0 \
+     --image-name metal3d/xmrig:latest \
+     --output=json \
+     --rules-path=/tmp/rules > xmrig-scan.json
+```
+
 ```
 # Extract the IOC array values.  From these, extract the values of the 'Matched Rule Name' key
 cat /tmp/xmrig-scan.json | jq '.IOC[] | ."Matched Rule Name"'

go.mod

@@ -5,6 +5,7 @@ go 1.21.0
 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc
 
 require (
+	github.com/VirusTotal/gyp v0.9.0
 	github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000
 	github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240807105002-4943c14781c5
 	github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240807105002-4943c14781c5
@@ -42,6 +43,7 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -61,6 +63,7 @@ require (
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect
 	go.opentelemetry.io/otel v1.27.0 // indirect

go.sum

@@ -10,6 +10,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/Microsoft/hcsshim v0.12.4 h1:Ev7YUMHAHoWNm+aDSPzc5W9s6E2jyL1szpVDJeZ/Rr4=
 github.com/Microsoft/hcsshim v0.12.4/go.mod h1:Iyl1WVpZzr+UkzjekHZbV8o5Z9ZkxNGx6CtY2Qg/JVQ=
+github.com/VirusTotal/gyp v0.9.0 h1:jhOBl93jfStmAcKLa/EcTmdPng5bn5kvJJZqQqJ5R4g=
+github.com/VirusTotal/gyp v0.9.0/go.mod h1:nmcW15dQ1657PmMcG9X/EZmp6rTQsyo9g8r6Cz1/AHc=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -84,13 +86,16 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -110,6 +115,12 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
 github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -155,11 +166,14 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
@@ -272,8 +286,11 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -25,16 +25,22 @@ package main
 // ------------------------------------------------------------------------------
 
 import (
+	"archive/tar"
 	"context"
+	"encoding/json"
+	"io"
+	"io/fs"
 	"os"
 	"os/signal"
 	"path"
+	"path/filepath"
 	"runtime"
 	"strconv"
 
 	"github.com/deepfence/YaraHunter/pkg/config"
 	"github.com/deepfence/YaraHunter/pkg/runner"
 	"github.com/deepfence/YaraHunter/pkg/server"
+	"github.com/deepfence/YaraHunter/pkg/threatintel"
 	cfg "github.com/deepfence/match-scanner/pkg/config"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
@@ -46,6 +52,13 @@ import (
 // and setup the session to start scanning for IOC
 // var session = core.GetSession()
 
+var (
+	version         string
+	checksumFile    = "checksum.txt"
+	sourceRuleFile  = "df-malware.json"
+	malwareRuleFile = "malware.yar"
+)
+
 func main() {
 	log.SetOutput(os.Stderr)
 	log.SetLevel(log.InfoLevel)
@@ -59,6 +72,8 @@ func main() {
 		},
 	})
 
+	log.Infof("version: %s", version)
+
 	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer cancel()
 
@@ -95,6 +110,11 @@ func main() {
 		go runner.ScheduleYaraHunterUpdater(ctx, runnerOpts)
 	}
 
+	// update rules required for cli mode
+	if *opts.SocketPath == "" {
+		updateRules(ctx, opts)
+	}
+
 	runner.StartYaraHunter(ctx, runnerOpts, config,
 		func(base *server.GRPCScannerServer) server.MalwareRPCServer {
 			return server.MalwareRPCServer{
@@ -105,4 +125,63 @@ func main() {
 		func(s grpc.ServiceRegistrar, impl any) {
 			pb.RegisterMalwareScannerServer(s, impl.(pb.MalwareScannerServer))
 		})
+
+}
+
+func updateRules(ctx context.Context, opts *config.Options) {
+	log.Infof("check and update malware rules")
+
+	listing, err := threatintel.FetchThreatIntelListing(ctx, version, *opts.Product, *opts.License)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	rulesInfo, err := listing.GetLatest(version, threatintel.MalwareDBType)
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Debug("rulesInfo: %+v", rulesInfo)
+
+	// make sure output rules directory exists
+	os.MkdirAll(*opts.RulesPath, fs.ModePerm)
+
+	// check if update required
+	if threatintel.SkipRulesUpdate(filepath.Join(*opts.RulesPath, checksumFile), rulesInfo.Checksum) {
+		log.Info("skip rules update")
+		return
+	}
+
+	log.Info("download new rules")
+	content, err := threatintel.DownloadFile(ctx, rulesInfo.URL)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Infof("rules file size: %d bytes", content.Len())
+
+	// write new checksum
+	if err := os.WriteFile(
+		filepath.Join(*opts.RulesPath, checksumFile), []byte(rulesInfo.Checksum), fs.ModePerm); err != nil {
+		log.Fatal(err)
+	}
+
+	// write rules file
+	outRuleFile := filepath.Join(*opts.RulesPath, malwareRuleFile)
+	threatintel.ProcessTarGz(content.Bytes(), sourceRuleFile, outRuleFile, processMalwareRules)
+}
+
+func processMalwareRules(header *tar.Header, reader io.Reader, outPath string) error {
+
+	var fb threatintel.FeedsBundle
+	if err := json.NewDecoder(reader).Decode(&fb); err != nil {
+		log.Error(err)
+		return err
+	}
+
+	if err := threatintel.ExportYaraRules(outPath, fb.ScannerFeeds.MalwareRules, fb.Extra); err != nil {
+		log.Error(err)
+		return err
+	}
+
+	return nil
 }

pkg/config/options.go

@@ -3,6 +3,13 @@ package config
 import (
 	"flag"
 	"os"
+
+	"github.com/deepfence/YaraHunter/utils"
+)
+
+var (
+	product string = utils.GetEnvOrDefault("DEEPFENCE_PRODUCT", "ThreatMapper")
+	license string = utils.GetEnvOrDefault("DEEPFENCE_LICENSE", "")
 )
 
 const (
@@ -37,6 +44,9 @@ type Options struct {
 	FailOnLowCount       *int
 	RulesListingURL      *string
 	EnableUpdater        *bool
+	Product              *string
+	Version              *string
+	License              *string
 }
 
 func ParseOptions() (*Options, error) {
@@ -67,6 +77,8 @@ func ParseOptions() (*Options, error) {
 		FailOnLowCount:       flag.Int("fail-on-low-count", -1, "Exit with status 1 if number of low malwares found is >= this value (Default: -1)"),
 		RulesListingURL:      flag.String("rules-listing-url", "https://threat-intel.deepfence.io/yara-rules/listing.json", "Deepfence threat intel yara rules listing (Default: threat-intel.deepfence.io/yara-rules/listing.json)"),
 		EnableUpdater:        flag.Bool("enable-updater", true, "Enable rules updater (Default: true)"),
+		Product:              flag.String("product", product, "Deepfence Product type can be ThreatMapper or ThreatStryker, also supports env var DEEPFENCE_PRODUCT"),
+		License:              flag.String("license", license, "TheratMapper or ThreatStryker license, also supports env var DEEPFENCE_LICENSE"),
 	}
 	flag.Parse()
 	return options, nil
@@ -99,5 +111,7 @@ func NewDefaultOptions() *Options {
 		ContainerID:          &emptyValue,
 		ContainerNS:          &emptyValue,
 		SocketPath:           &emptyValue,
+		Product:              &product,
+		License:              &license,
 	}
 }

pkg/threatintel/feeds.go

@@ -0,0 +1,38 @@
+package threatintel
+
+type FeedsBundle struct {
+	Version      string       `json:"version"`
+	CreatedAt    int64        `json:"created_at"`
+	ScannerFeeds ScannerFeeds `json:"scanner_feeds"`
+	TracerFeeds  TracerFeeds  `json:"tracer_feeds"`
+	Extra        []string     `json:"extra"`
+}
+
+type ScannerFeeds struct {
+	VulnerabilityRules   []DeepfenceRule `json:"vulnerability_rules"`
+	SecretRules          []DeepfenceRule `json:"secret_rules"`
+	MalwareRules         []DeepfenceRule `json:"malware_rules"`
+	ComplianceRules      []DeepfenceRule `json:"compliance_rules"`
+	CloudComplianceRules []DeepfenceRule `json:"cloud_compliance_rules"`
+}
+
+type TracerFeeds struct {
+	NetworkRules      []DeepfenceRule `json:"network_rules"`
+	FilesystemRules   []DeepfenceRule `json:"filesystem_rules"`
+	ProcessRules      []DeepfenceRule `json:"process_rules"`
+	ExternalArtefacts []Artefact      `json:"external_artefacts"`
+}
+
+type Artefact struct {
+	Name    string `json:"name"`
+	Type    string `json:"type"`
+	Content []byte `json:"content"`
+}
+
+type DeepfenceRule struct {
+	RuleID      string `json:"rule_id"`
+	Type        string `json:"type"`
+	Payload     string `json:"payload"`
+	Severity    string `json:"severity"`
+	Description string `json:"description"`
+}