From bc8c70cf15cd7875e1416ad5e180532cc6451194 Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Tue, 29 Oct 2024 09:30:09 +0000 Subject: [PATCH] add support for rules download in cli mode --- Dockerfile | 6 +- Makefile | 2 +- README.md | 22 +++++- go.mod | 3 + go.sum | 19 ++++- main.go | 79 +++++++++++++++++++ pkg/config/options.go | 14 ++++ pkg/threatintel/feeds.go | 38 +++++++++ pkg/threatintel/listing.go | 61 +++++++++++++++ pkg/threatintel/threatintel.go | 79 +++++++++++++++++++ pkg/threatintel/utils.go | 137 +++++++++++++++++++++++++++++++++ utils/utils.go | 8 ++ 12 files changed, 460 insertions(+), 8 deletions(-) create mode 100644 pkg/threatintel/feeds.go create mode 100644 pkg/threatintel/listing.go create mode 100644 pkg/threatintel/threatintel.go create mode 100644 pkg/threatintel/utils.go diff --git a/Dockerfile b/Dockerfile index d22b56f..55b3bbd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,10 +52,7 @@ RUN cd /root && wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.3.2 WORKDIR /home/deepfence/src/YaraHunter COPY . . -RUN make clean \ - && make all \ - && cd /home/deepfence \ - && git clone https://github.com/deepfence/yara-rules +RUN make clean && make all FROM debian:bookworm @@ -95,7 +92,6 @@ EOF RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y libgpgme-dev libdevmapper-dev WORKDIR /home/deepfence/usr -COPY --from=builder /home/deepfence/yara-rules . COPY --from=builder /usr/local/yara.tar.gz /usr/local/yara.tar.gz COPY --from=builder /home/deepfence/src/YaraHunter/YaraHunter . COPY --from=builder /home/deepfence/src/YaraHunter/config.yaml . diff --git a/Makefile b/Makefile index 181efcf..7caef83 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ vendor: go.mod go mod vendor yarahunter: vendor $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/**/*.go - CGO_LDFLAGS="-ljansson -lcrypto -lmagic" PKG_CONFIG_PATH=/usr/local/yara/lib/pkgconfig:$(PKG_CONFIG_PATH) go build -buildmode=pie -ldflags="-s -w -extldflags=-static" -buildvcs=false -v . + CGO_LDFLAGS="-ljansson -lcrypto -lmagic" PKG_CONFIG_PATH=/usr/local/yara/lib/pkgconfig:$(PKG_CONFIG_PATH) go build -buildmode=pie -ldflags="-s -w -extldflags=-static -X 'main.version=$(DF_IMG_TAG)'" -buildvcs=false -v . .PHONY: clean bootstrap diff --git a/README.md b/README.md index 83c6804..d96b404 100644 --- a/README.md +++ b/README.md @@ -49,17 +49,37 @@ Pull the image that needs to be scanned for example `metal3d/xmrig` and scan it: ``` docker pull metal3d/xmrig +``` + +Set Product and Licence and scan it: +``` docker run -i --rm --name=deepfence-yarahunter \ + -e DEEPFENCE_PRODUCT= \ + -e DEEPFENCE_LICENSE= \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/home/deepfence/output \ - quay.io/deepfenceio/deepfence_malware_scanner_ce:2.3.0 \ + quay.io/deepfenceio/deepfence_malware_scanner_ce:3.0.0 \ --image-name metal3d/xmrig:latest \ --output=json > xmrig-scan.json ``` This returns, among other things, clear indication of the presence of XMRig. Note that we store the output (`xmrig-scan.json`) for quick and easy manipulation: +Rules can also be cached to use next run by mounting a seperate path and passing `rules-path` argument +``` +docker run -i --rm --name=deepfence-yarahunter \ + -e DEEPFENCE_PRODUCT= \ + -e DEEPFENCE_LICENSE= \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/home/deepfence/output \ + -v /tmp/rules:/tmp/rules \ + quay.io/deepfenceio/deepfence_malware_scanner_ce:3.0.0 \ + --image-name metal3d/xmrig:latest \ + --output=json \ + --rules-path=/tmp/rules > xmrig-scan.json +``` + ``` # Extract the IOC array values. From these, extract the values of the 'Matched Rule Name' key cat /tmp/xmrig-scan.json | jq '.IOC[] | ."Matched Rule Name"' diff --git a/go.mod b/go.mod index 5a3da9b..f8266e2 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21.0 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc require ( + github.com/VirusTotal/gyp v0.9.0 github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240807105002-4943c14781c5 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240807105002-4943c14781c5 @@ -42,6 +43,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -61,6 +63,7 @@ require ( github.com/opencontainers/selinux v1.11.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect go.opentelemetry.io/otel v1.27.0 // indirect diff --git a/go.sum b/go.sum index 4bba840..54a0c43 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.12.4 h1:Ev7YUMHAHoWNm+aDSPzc5W9s6E2jyL1szpVDJeZ/Rr4= github.com/Microsoft/hcsshim v0.12.4/go.mod h1:Iyl1WVpZzr+UkzjekHZbV8o5Z9ZkxNGx6CtY2Qg/JVQ= +github.com/VirusTotal/gyp v0.9.0 h1:jhOBl93jfStmAcKLa/EcTmdPng5bn5kvJJZqQqJ5R4g= +github.com/VirusTotal/gyp v0.9.0/go.mod h1:nmcW15dQ1657PmMcG9X/EZmp6rTQsyo9g8r6Cz1/AHc= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -84,6 +86,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -91,6 +95,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -110,6 +115,12 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -155,11 +166,14 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -272,8 +286,11 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 8d1066e..2c4e13c 100644 --- a/main.go +++ b/main.go @@ -25,16 +25,22 @@ package main // ------------------------------------------------------------------------------ import ( + "archive/tar" "context" + "encoding/json" + "io" + "io/fs" "os" "os/signal" "path" + "path/filepath" "runtime" "strconv" "github.com/deepfence/YaraHunter/pkg/config" "github.com/deepfence/YaraHunter/pkg/runner" "github.com/deepfence/YaraHunter/pkg/server" + "github.com/deepfence/YaraHunter/pkg/threatintel" cfg "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -46,6 +52,13 @@ import ( // and setup the session to start scanning for IOC // var session = core.GetSession() +var ( + version string + checksumFile = "checksum.txt" + sourceRuleFile = "df-malware.json" + malwareRuleFile = "malware.yar" +) + func main() { log.SetOutput(os.Stderr) log.SetLevel(log.InfoLevel) @@ -59,6 +72,8 @@ func main() { }, }) + log.Infof("version: %s", version) + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() @@ -95,6 +110,11 @@ func main() { go runner.ScheduleYaraHunterUpdater(ctx, runnerOpts) } + // update rules required for cli mode + if *opts.SocketPath == "" { + updateRules(ctx, opts) + } + runner.StartYaraHunter(ctx, runnerOpts, config, func(base *server.GRPCScannerServer) server.MalwareRPCServer { return server.MalwareRPCServer{ @@ -105,4 +125,63 @@ func main() { func(s grpc.ServiceRegistrar, impl any) { pb.RegisterMalwareScannerServer(s, impl.(pb.MalwareScannerServer)) }) + +} + +func updateRules(ctx context.Context, opts *config.Options) { + log.Infof("check and update malware rules") + + listing, err := threatintel.FetchThreatIntelListing(ctx, version, *opts.Product, *opts.License) + if err != nil { + log.Fatal(err) + } + + rulesInfo, err := listing.GetLatest(version, threatintel.MalwareDBType) + if err != nil { + log.Fatal(err) + } + log.Debug("rulesInfo: %+v", rulesInfo) + + // make sure output rules directory exists + os.MkdirAll(*opts.RulesPath, fs.ModePerm) + + // check if update required + if threatintel.SkipRulesUpdate(filepath.Join(*opts.RulesPath, checksumFile), rulesInfo.Checksum) { + log.Info("skip rules update") + return + } + + log.Info("download new rules") + content, err := threatintel.DownloadFile(ctx, rulesInfo.URL) + if err != nil { + log.Fatal(err) + } + + log.Infof("rules file size: %d bytes", content.Len()) + + // write new checksum + if err := os.WriteFile( + filepath.Join(*opts.RulesPath, checksumFile), []byte(rulesInfo.Checksum), fs.ModePerm); err != nil { + log.Fatal(err) + } + + // write rules file + outRuleFile := filepath.Join(*opts.RulesPath, malwareRuleFile) + threatintel.ProcessTarGz(content.Bytes(), sourceRuleFile, outRuleFile, processMalwareRules) +} + +func processMalwareRules(header *tar.Header, reader io.Reader, outPath string) error { + + var fb threatintel.FeedsBundle + if err := json.NewDecoder(reader).Decode(&fb); err != nil { + log.Error(err) + return err + } + + if err := threatintel.ExportYaraRules(outPath, fb.ScannerFeeds.MalwareRules, fb.Extra); err != nil { + log.Error(err) + return err + } + + return nil } diff --git a/pkg/config/options.go b/pkg/config/options.go index ace0223..60b9acc 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -3,6 +3,13 @@ package config import ( "flag" "os" + + "github.com/deepfence/YaraHunter/utils" +) + +var ( + product string = utils.GetEnvOrDefault("DEEPFENCE_PRODUCT", "ThreatMapper") + license string = utils.GetEnvOrDefault("DEEPFENCE_LICENSE", "") ) const ( @@ -37,6 +44,9 @@ type Options struct { FailOnLowCount *int RulesListingURL *string EnableUpdater *bool + Product *string + Version *string + License *string } func ParseOptions() (*Options, error) { @@ -67,6 +77,8 @@ func ParseOptions() (*Options, error) { FailOnLowCount: flag.Int("fail-on-low-count", -1, "Exit with status 1 if number of low malwares found is >= this value (Default: -1)"), RulesListingURL: flag.String("rules-listing-url", "https://threat-intel.deepfence.io/yara-rules/listing.json", "Deepfence threat intel yara rules listing (Default: threat-intel.deepfence.io/yara-rules/listing.json)"), EnableUpdater: flag.Bool("enable-updater", true, "Enable rules updater (Default: true)"), + Product: flag.String("product", product, "Deepfence Product type can be ThreatMapper or ThreatStryker, also supports env var DEEPFENCE_PRODUCT"), + License: flag.String("license", license, "TheratMapper or ThreatStryker license, also supports env var DEEPFENCE_LICENSE"), } flag.Parse() return options, nil @@ -99,5 +111,7 @@ func NewDefaultOptions() *Options { ContainerID: &emptyValue, ContainerNS: &emptyValue, SocketPath: &emptyValue, + Product: &product, + License: &license, } } diff --git a/pkg/threatintel/feeds.go b/pkg/threatintel/feeds.go new file mode 100644 index 0000000..b19a4ef --- /dev/null +++ b/pkg/threatintel/feeds.go @@ -0,0 +1,38 @@ +package threatintel + +type FeedsBundle struct { + Version string `json:"version"` + CreatedAt int64 `json:"created_at"` + ScannerFeeds ScannerFeeds `json:"scanner_feeds"` + TracerFeeds TracerFeeds `json:"tracer_feeds"` + Extra []string `json:"extra"` +} + +type ScannerFeeds struct { + VulnerabilityRules []DeepfenceRule `json:"vulnerability_rules"` + SecretRules []DeepfenceRule `json:"secret_rules"` + MalwareRules []DeepfenceRule `json:"malware_rules"` + ComplianceRules []DeepfenceRule `json:"compliance_rules"` + CloudComplianceRules []DeepfenceRule `json:"cloud_compliance_rules"` +} + +type TracerFeeds struct { + NetworkRules []DeepfenceRule `json:"network_rules"` + FilesystemRules []DeepfenceRule `json:"filesystem_rules"` + ProcessRules []DeepfenceRule `json:"process_rules"` + ExternalArtefacts []Artefact `json:"external_artefacts"` +} + +type Artefact struct { + Name string `json:"name"` + Type string `json:"type"` + Content []byte `json:"content"` +} + +type DeepfenceRule struct { + RuleID string `json:"rule_id"` + Type string `json:"type"` + Payload string `json:"payload"` + Severity string `json:"severity"` + Description string `json:"description"` +} diff --git a/pkg/threatintel/listing.go b/pkg/threatintel/listing.go new file mode 100644 index 0000000..8d71920 --- /dev/null +++ b/pkg/threatintel/listing.go @@ -0,0 +1,61 @@ +package threatintel + +import ( + "sort" + "time" +) + +const ( + MalwareDBType = "malware" + SecretDBType = "secret" +) + +type Listing struct { + Available map[string][]Entry `json:"available"` +} + +type Entry struct { + Built time.Time `json:"built"` + Version string `json:"version"` + Type string `json:"type"` + URL string `json:"url"` + Checksum string `json:"checksum"` +} + +func (l *Listing) GetLatest(version, dbType string) (Entry, error) { + + entries := []Entry{} + + for _, e := range l.Available[version] { + if e.Type == dbType { + entries = append(entries, e) + } + } + + sort.Slice(entries, func(i, j int) bool { + return entries[i].Built.Before(entries[j].Built) + }) + + if len(entries) >= 1 { + return entries[len(entries)-1], nil + } + + return Entry{}, ErrDatabaseNotFound + +} + +func (l *Listing) GetLatestN(version string, dbType ...string) ([]Entry, error) { + + entries := []Entry{} + + for _, e := range dbType { + dbinfo, err := l.GetLatest(version, e) + if err != nil && err != ErrDatabaseNotFound { + return entries, err + } + entries = append(entries, dbinfo) + } + + return entries, nil + +} diff --git a/pkg/threatintel/threatintel.go b/pkg/threatintel/threatintel.go new file mode 100644 index 0000000..0b8dc67 --- /dev/null +++ b/pkg/threatintel/threatintel.go @@ -0,0 +1,79 @@ +package threatintel + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" + + "github.com/deepfence/YaraHunter/utils" + log "github.com/sirupsen/logrus" +) + +var ( + threatIntelURL = "https://threat-intel.deepfence.io/threat-intel/listing.json" + threatIntelTest = utils.GetEnvOrDefault("DEEPFENCE_THREAT_INTEL_TEST", "false") == "true" +) + +var ErrDatabaseNotFound = errors.New("database type not found") + +func FetchThreatIntelListing(ctx context.Context, version, project, license string) (Listing, error) { + + var listing Listing + + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + hc := http.Client{ + Timeout: 10 * time.Second, + Transport: tr, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, threatIntelURL, nil) + if err != nil { + log.Error("failed to construct new http request") + return listing, err + } + + req.Header.Set("x-license-key", license) + + q := req.URL.Query() + q.Add("version", version) + q.Add("product", project) + if threatIntelTest { + q.Add("test", "true") + } + req.URL.RawQuery = q.Encode() + + log.Debugf("query threatintel at %s", req.URL.String()) + + resp, err := hc.Do(req) + if err != nil { + log.Error("failed http request") + return listing, err + } + + if resp.StatusCode != http.StatusOK { + return listing, fmt.Errorf("%d invaid response code", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Error("failed read response body") + return listing, err + } + defer resp.Body.Close() + + if err := json.Unmarshal(body, &listing); err != nil { + log.Error("failed to decode response body") + return listing, err + } + + return listing, nil + +} diff --git a/pkg/threatintel/utils.go b/pkg/threatintel/utils.go new file mode 100644 index 0000000..99226bb --- /dev/null +++ b/pkg/threatintel/utils.go @@ -0,0 +1,137 @@ +package threatintel + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "context" + "crypto/tls" + "encoding/base64" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "strings" + "time" + + "github.com/VirusTotal/gyp" + log "github.com/sirupsen/logrus" +) + +func ExportYaraRules(outFile string, rules []DeepfenceRule, extra []string) error { + + file, err := os.OpenFile(outFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.ModePerm) + if err != nil { + log.Errorf("failed to open file: %s, skipping", err) + return err + } + defer file.Close() + + for i := range extra { + file.WriteString(fmt.Sprintf("import \"%s\"\n", extra[i])) + } + + for _, rule := range rules { + decoded, err := base64.StdEncoding.DecodeString(rule.Payload) + if err != nil { + log.Errorf("err on base64 decode: %v", err) + continue + } + rs, err := gyp.ParseString(string(decoded)) + if err != nil { + log.Errorf("err on parse: %v", err) + continue + } + for _, r := range rs.Rules { + r.WriteSource(file) + } + } + + return nil +} + +func DownloadFile(ctx context.Context, url string) (*bytes.Buffer, error) { + + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + + client := http.Client{Timeout: 600 * time.Second} + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("bad status: %s", resp.Status) + } + + var out bytes.Buffer + _, err = io.Copy(bufio.NewWriter(&out), resp.Body) + if err != nil { + return nil, err + } + + return &out, nil +} + +func ProcessTarGz(content []byte, sourceFile string, outPath string, + processFile func(header *tar.Header, reader io.Reader, outPath string) error) error { + // Uncompress the gzipped content + gzipReader, err := gzip.NewReader(bytes.NewReader(content)) + if err != nil { + return fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzipReader.Close() + + // Create a tar reader to read the uncompressed data + tarReader := tar.NewReader(gzipReader) + + // Iterate over the files in the tar archive + for { + header, err := tarReader.Next() + if err == io.EOF { + break // End of tar archive + } + if err != nil { + return fmt.Errorf("failed to read tar file: %w", err) + } + + // skip some files + if header.FileInfo().IsDir() { + continue + } + + if !strings.Contains(header.Name, sourceFile) { + continue + } + + // Run the provided callback function on the current file + if err := processFile(header, tarReader, outPath); err != nil { + return fmt.Errorf("failed to process file %s: %w", header.Name, err) + } + } + + return nil +} + +func SkipRulesUpdate(checksumFilePath, checksum string) bool { + sum, err := os.ReadFile(checksumFilePath) + if err != nil && os.IsNotExist(err) { + return false + } + + if string(sum) == checksum { + return true + } + + return false +} diff --git a/utils/utils.go b/utils/utils.go index fdd6805..0c5c446 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -121,3 +121,11 @@ func GetDfInstallDir() string { return "" } } + +func GetEnvOrDefault(envVar string, defaultValue string) string { + envValue := os.Getenv(envVar) + if len(envValue) == 0 { + return defaultValue + } + return envValue +}