diff --git a/README.md b/README.md index 3d3c794..9fc1c9e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The goal of this application is to allow the user to pull FFS logs from Code42's ## Important Notes: -- This application, while working, is still considered under development, as it has not be thoroughly tested. There are likely bugs and random errors/panics which may occur. If they do, please report them so that they can be fixed. +- This application, while working, is still considered under development, as it has not been thoroughly tested. There are likely bugs and random errors/panics which may occur. If they do, please report them so that they can be fixed. - The slowest part of this application is the downloading of the logs. You should try to optimize your queries as much as possible. Limitations: @@ -14,8 +14,8 @@ Limitations: Code42 Crashplan FFS API has limitations like most APIs, these limitations affect the functionality of the application as followed: 1. 120 Queries per minute, any additional queries will be dropped. (never actually bothered to test if/how this limit is actually enforced) -1. 200,000 results returned per query. This limitation is kind of annoying to handle as there is no easy way to handle it. The API does not support paging and the only way to figure out how many results there is for a query is to first query, count, then if over 200,000 results, break up the query into smaller time increments and perform multiple queries to get all of the results. -1. The application only supports the /v1/fileevent/export API endpoint currently. This has to do with how the highly limited functionality of the /v1/fileevent endpoint which isn't well documented. + 1. It is recommended to set the page size of the query to at least 1000, but less than 10000 to reduce the number of queries required +1. The application only supports the use of the /v1/fileevent API endpoint, which provides a JSON formatted output. ## Install @@ -33,7 +33,7 @@ $ /path/to/output/location/crashplan-ffs-puller --config=/path/to/config.json ### Precompiled Binaries -These are found attached to each official release. Currently only Windows amd64 and Linux amd64 binaries will be released. As the application progresses, or more needs come up, more binaries will be added. +These are found attached to each official release. Currently, only Windows amd64 and Linux amd64 binaries will be released. As the application progresses or more needs arise, more binaries will be added. ### Docker @@ -73,7 +73,7 @@ Currently, only JSON formatted configuration files are accepted, in the future Y ``` { "authURI": "https://www.crashplan.com/c42api/v3/auth/jwt?useBody=true", #This is the URI which has the Code42 authentication endpoint. - "ffsURI": "https://forensicsearch-default.prod.ffs.us2.code42.com/forensic-search/queryservice/api/v1/fileevent/export", #This is the URI which exposes the FFS API. Note: Currently only supports the fileevent/export endpoint. + "ffsURI": "https://forensicsearch-default.prod.ffs.us2.code42.com/forensic-search/queryservice/api/v1/fileevent", #This is the URI which exposes the FFS API. Note: Currently only supports the /fileevent endpoint. "ffsQueries": [{ #This is an area of FFS Queries + additional information. "name": "example_query_1", #Query name, must be unique. "username": "example@example.com", #Username, must be an email address. @@ -103,7 +103,8 @@ Currently, only JSON formatted configuration files are accepted, in the future Y ], "filterClause": "AND" } - ] + ], + "pgSize": 1000 }, "outputType": "elastic", #Output type, supports either file, elastic, logstash "outputLocation": "/path/to/output", #This is needed even if not using file output type, as there are stateful files which need to be written and stored. @@ -193,7 +194,7 @@ In the above configuration there are some important notes to know about the FFS Note: I have not tested out all possible queries in this application, if you come across a query which does not work, let me know and I will try to get it working. -### Elasticsearch Integration +### Elasticsearch Integration (WIP) If you are using the elastic output type there are a few important things to understand. @@ -256,5 +257,4 @@ If you have any ideas for other metrics you feel may be useful, feel free to ope ##TODOs (Maybe) 1. Add ability to use yaml/yml configuration files. -2. Add the ability to use the regular json FFS API endpoint (/fileevent). 3. Add some sort of file hash lookup that could provide threat intelligence (tried this with [OTX](https://www.alienvault.com/open-threat-exchange) before and failed pretty bad, may look at revisiting). \ No newline at end of file diff --git a/VERSION b/VERSION index b300caa..79a2734 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.4 \ No newline at end of file +0.5.0 \ No newline at end of file diff --git a/eventOutput/fileHandler.go b/eventOutput/fileHandler.go index b8f5a50..a9052e7 100644 --- a/eventOutput/fileHandler.go +++ b/eventOutput/fileHandler.go @@ -15,42 +15,47 @@ import ( ) type FFSEvent struct { - ffs.FileEvent + ffs.JsonFileEvent *ip_api.Location `json:",omitempty"` GeoLocation *Location `json:"geoPoint,omitempty"` } type Code42 struct { - Event *Code42Event `json:"event,omitempty"` - InsertionTimestamp *time.Time `json:"insertion_timestamp,omitempty"` - File *Code42File `json:"file,omitempty"` - Device *Code42Device `json:"device,omitempty"` - OsHostName string `json:"os_host_name,omitempty"` - DomainName string `json:"domain_name,omitempty"` - PublicIpAddress string `json:"public_ip_address,omitempty"` - PrivateIpAddresses []string `json:"private_ip_addresses,omitempty"` - Actor string `json:"actor,omitempty"` - DirectoryId []string `json:"directory_id,omitempty"` - Source string `json:"source,omitempty"` - Url *URL `json:"url,omitempty"` - Shared *bool `json:"shared,omitempty"` - SharedWith []string `json:"shared_with,omitempty"` - SharingTypeAdded []string `json:"sharing_type_added,omitempty"` - CloudDriveId string `json:"cloud_drive_id,omitempty"` - DetectionSourceAlias string `json:"detection_source_alias,omitempty"` - Exposure []string `json:"exposure,omitempty"` - Process *Code42Process `json:"process,omitempty"` - Tab *Code42Tab `json:"tab,omitempty"` - RemovableMedia *Code42RemovableMedia `json:"removable_media,omitempty"` - SyncDestination string `json:"sync_destination,omitempty"` - SyncDestinationUsername string `json:"sync_destination_username,omitempty"` - EmailDlp *Code42EmailDlp `json:"email_dlp,omitempty"` - OutsideActiveHours *bool `json:"outside_active_hours,omitempty"` - Print *Code42Print `json:"print,omitempty"` - RemoteActivity string `json:"remote_activity,omitempty"` - Trusted *bool `json:"trusted,omitempty"` - LoggedInOperatingSystemUser string `json:"logged_in_operating_system_user,omitempty"` - Destination *Code42Destination `json:"destination,omitempty"` + Event *Code42Event `json:"event,omitempty"` + InsertionTimestamp *time.Time `json:"insertion_timestamp,omitempty"` + File *Code42File `json:"file,omitempty"` + Device *Code42Device `json:"device,omitempty"` + OsHostName string `json:"os_host_name,omitempty"` + DomainName string `json:"domain_name,omitempty"` + PublicIpAddress string `json:"public_ip_address,omitempty"` + PrivateIpAddresses []string `json:"private_ip_addresses,omitempty"` + Actor string `json:"actor,omitempty"` + DirectoryId []string `json:"directory_id,omitempty"` + Source string `json:"source,omitempty"` + Url *URL `json:"url,omitempty"` + Shared string `json:"shared,omitempty"` + SharedWith []ffs.SharedWith `json:"shared_with,omitempty"` + SharingTypeAdded []string `json:"sharing_type_added,omitempty"` + CloudDriveId string `json:"cloud_drive_id,omitempty"` + DetectionSourceAlias string `json:"detection_source_alias,omitempty"` + Exposure []string `json:"exposure,omitempty"` + Process *Code42Process `json:"process,omitempty"` + RemovableMedia *Code42RemovableMedia `json:"removable_media,omitempty"` + SyncDestination string `json:"sync_destination,omitempty"` + SyncDestinationUsername []string `json:"sync_destination_username,omitempty"` + EmailDlp *Code42EmailDlp `json:"email_dlp,omitempty"` + OutsideActiveHours *bool `json:"outside_active_hours,omitempty"` + Print *Code42Print `json:"print,omitempty"` + RemoteActivity string `json:"remote_activity,omitempty"` + Trusted *bool `json:"trusted,omitempty"` + OperatingSystemUser string `json:"operating_system_user,omitempty"` + Destination *Code42Destination `json:"destination,omitempty"` + Tabs []Code42TabTab `json:"tabs,omitempty"` +} + +type Code42TabTab struct { + Title string `json:"title,omitempty"` + Url *URL `json:"url,omitempty"` } type Code42Event struct { @@ -64,17 +69,17 @@ type Code42File struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` Category string `json:"category,omitempty"` - IdentifiedExtensionCategory string `json:"identified_extension_category,omitempty"` - CurrentExtensionCategory string `json:"current_extension_category,omitempty"` - Size *int `json:"size,omitempty"` - Owner []string `json:"owner,omitempty"` + MimeTypeByBytes string `json:"mime_type_by_bytes,omitempty"` + MimeTypeByExtension string `json:"mime_type_by_extension,omitempty"` + Size *int64 `json:"size,omitempty"` + Owner string `json:"owner,omitempty"` Hash *Hash `json:"hash,omitempty"` - CreatedTimestamp *time.Time `json:"created_timestamp,omitempty"` + CreateTimestamp *time.Time `json:"create_timestamp,omitempty"` ModifyTimestamp *time.Time `json:"modify_timestamp,omitempty"` Id string `json:"id,omitempty"` IdentifiedExtensionMIMEType string `json:"identified_extension_mime_type,omitempty"` CurrentExtensionMIMEType string `json:"current_extension_mime_type,omitempty"` - SuspiciousFileTypeMismatch *bool `json:"suspicious_file_type_mismatch,omitempty"` + MimeTypeMismatch *bool `json:"mime_type_mismatch,omitempty"` } type Code42Device struct { @@ -85,19 +90,17 @@ type Code42Device struct { type Code42Tab struct { WindowTitle string `json:"window_title,omitempty"` Url *URL `json:"url,omitempty"` - Titles []string `json:"titles,omitempty"` - Urls []URL `json:"urls,omitempty"` } type Code42RemovableMedia struct { - Vendor string `json:"vendor,omitempty"` - Name string `json:"name,omitempty"` - SerialNumber string `json:"serial_number,omitempty"` - Capacity *int `json:"capacity,omitempty"` - BusType string `json:"bus_type,omitempty"` - MediaName string `json:"media_name,omitempty"` - VolumeName string `json:"volume_name,omitempty"` - PartitionId string `json:"partition_id,omitempty"` + Vendor string `json:"vendor,omitempty"` + Name string `json:"name,omitempty"` + SerialNumber string `json:"serial_number,omitempty"` + Capacity *int64 `json:"capacity,omitempty"` + BusType string `json:"bus_type,omitempty"` + MediaName string `json:"media_name,omitempty"` + VolumeName []string `json:"volume_name,omitempty"` + PartitionId []string `json:"partition_id,omitempty"` } type Code42EmailDlp struct { @@ -109,9 +112,8 @@ type Code42EmailDlp struct { } type Code42Print struct { - JobName string `json:"job_name,omitempty"` - PrinterName string `json:"name,omitempty"` - PrintedFilesBackupPath string `json:"printed_files_backup_path,omitempty"` + JobName string `json:"job_name,omitempty"` + PrinterName string `json:"name,omitempty"` } type Code42Destination struct { @@ -165,12 +167,12 @@ type File struct { Created *time.Time `json:"created,omitempty"` Directory []string `json:"directory,omitempty"` Extension string `json:"extension,omitempty"` - MimeType string `json:"mime_type,omitempty"` + MimeType []string `json:"mime_type,omitempty"` Mtime *time.Time `json:"mtime,omitempty"` Name string `json:"name,omitempty"` - Owner []string `json:"owner,omitempty"` + Owner string `json:"owner,omitempty"` Path string `json:"path,omitempty"` - Size *int `json:"size,omitempty"` + Size *int64 `json:"size,omitempty"` Type string `json:"type,omitempty"` Hash *Hash `json:"hash,omitempty"` } diff --git a/ffsEvent/ffsFetcher.go b/ffsEvent/ffsFetcher.go index 850e78d..6143a01 100644 --- a/ffsEvent/ffsFetcher.go +++ b/ffsEvent/ffsFetcher.go @@ -9,7 +9,7 @@ import ( "github.com/BenB196/crashplan-ffs-puller/elasticsearch" "github.com/BenB196/crashplan-ffs-puller/eventOutput" "github.com/BenB196/crashplan-ffs-puller/promMetrics" - ip_api "github.com/BenB196/ip-api-go-pkg" + "github.com/BenB196/ip-api-go-pkg" "github.com/olivere/elastic/v7" "log" "path/filepath" @@ -68,7 +68,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg notInProgressTime := time.Now() - fileEvents, err := ffs.GetFileEvents(authData, configuration.FFSURI, query.Query) + fileEvents, _, err := ffs.GetJsonFileEvents(authData, configuration.FFSURI, query.Query, "", configuration.Debugging) getFileEventsTime := time.Now() @@ -134,20 +134,76 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg eventType = "deletion" } + //eventTimestamp + if len(strings.Split(ffsEvent.EventTimestamp, ".")) != 2 { + ffsEvent.EventTimestamp = ffsEvent.EventTimestamp + ".000" + } + eventTimestamp, err := time.Parse("2006-01-02 15:04:05.000", strings.Replace(strings.Replace(ffsEvent.EventTimestamp, "T", " ", -1), "Z", "", -1)) + + if err != nil { + panic(err) + } + + //insertionTimestamp + + if len(strings.Split(ffsEvent.InsertionTimestamp, ".")) != 2 { + ffsEvent.InsertionTimestamp = ffsEvent.InsertionTimestamp + ".000" + } + + insertionTimeSplit := strings.Split(ffsEvent.InsertionTimestamp, ".") + insertionTimestampLength := len(insertionTimeSplit[1]) + if insertionTimestampLength > 4 { + ffsEvent.InsertionTimestamp = insertionTimeSplit[0] + "." + insertionTimeSplit[1][0:3] + } + + insertionTimestamp, err := time.Parse("2006-01-02 15:04:05.000", strings.Replace(strings.Replace(ffsEvent.InsertionTimestamp, "T", " ", -1), "Z", "", -1)) + if err != nil { + panic(err) + } + + //creationTimestamp + var createTimestamp time.Time + if ffsEvent.CreateTimestamp != "" { + if len(strings.Split(ffsEvent.CreateTimestamp, ".")) != 2 { + ffsEvent.CreateTimestamp = ffsEvent.CreateTimestamp + ".000" + } + + createTimestamp, err = time.Parse("2006-01-02 15:04:05.000", strings.Replace(strings.Replace(ffsEvent.CreateTimestamp, "T", " ", -1), "Z", "", -1)) + + if err != nil { + panic(err) + } + } + + //modifyTimestamp + var modifyTimestamp time.Time + if ffsEvent.ModifyTimestamp != "" { + if len(strings.Split(ffsEvent.ModifyTimestamp, ".")) != 2 { + ffsEvent.ModifyTimestamp = ffsEvent.ModifyTimestamp + ".000" + } + + modifyTimestamp, err = time.Parse("2006-01-02 15:04:05.000", strings.Replace(strings.Replace(ffsEvent.ModifyTimestamp, "T", " ", -1), "Z", "", -1)) + + if err != nil { + panic(err) + } + } + + event := &eventOutput.Event{ Action: ffsEvent.EventType, Category: "file", - Created: ffsEvent.EventTimestamp, + Created: &eventTimestamp, Dataset: "code42.ffs", Id: ffsEvent.EventId, - Ingested: ffsEvent.InsertionTimestamp, + Ingested: &insertionTimestamp, Kind: "event", Module: "code42", Type: eventType, } //@timestamp - timestamp := ffsEvent.EventTimestamp + timestamp := eventTimestamp //file fields hash := &eventOutput.Hash{ @@ -172,25 +228,25 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg Path: ffsEvent.FilePath, Name: ffsEvent.FileName, Type: fileType, - Extension: filepath.Ext(ffsEvent.FileName), + Extension: strings.Replace(filepath.Ext(ffsEvent.FileName),".","",-1), Size: ffsEvent.FileSize, Owner: ffsEvent.FileOwner, Hash: hash, - Created: ffsEvent.CreatedTimestamp, - Mtime: ffsEvent.ModifyTimestamp, + Created: &createTimestamp, + Mtime: &modifyTimestamp, Directory: ffsEvent.DirectoryId, - MimeType: ffsEvent.CurrentExtensionMIMEType, + MimeType: []string{ffsEvent.MimeTypeByBytes, ffsEvent.MimeTypeByExtension}, } //user fields var user *eventOutput.User - if ffsEvent.DeviceUsername == "NAME_NOT_AVAILABLE" { + if ffsEvent.DeviceUserName == "NAME_NOT_AVAILABLE" { name := "" domain := "" if strings.Contains(ffsEvent.Actor, "@") { name = strings.Split(ffsEvent.Actor, "@")[0] - domain = strings.Split(ffsEvent.DeviceUsername, "@")[1] + domain = strings.Split(ffsEvent.DeviceUserName, "@")[1] } user = &eventOutput.User{ @@ -202,20 +258,20 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg } else { name := "" domain := "" - if strings.Contains(ffsEvent.DeviceUsername, "@") { - name = strings.Split(ffsEvent.DeviceUsername, "@")[0] - domain = strings.Split(ffsEvent.DeviceUsername, "@")[1] + if strings.Contains(ffsEvent.DeviceUserName, "@") { + name = strings.Split(ffsEvent.DeviceUserName, "@")[0] + domain = strings.Split(ffsEvent.DeviceUserName, "@")[1] } user = &eventOutput.User{ - Email: ffsEvent.DeviceUsername, + Email: ffsEvent.DeviceUserName, Id: ffsEvent.UserUid, Name: name, Domain: domain, } } - if ffsEvent.LoggedInOperatingSystemUser != "" && ffsEvent.LoggedInOperatingSystemUser != "NAME_NOT_AVAILABLE" { - user.Id = ffsEvent.LoggedInOperatingSystemUser + if ffsEvent.OperatingSystemUser != "" && ffsEvent.OperatingSystemUser != "NAME_NOT_AVAILABLE" { + user.Id = ffsEvent.OperatingSystemUser } if *user == (eventOutput.User{}) { @@ -281,7 +337,7 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg host := &eventOutput.Host{ Id: ffsEvent.DeviceUid, - Name: ffsEvent.OsHostname, + Name: ffsEvent.OsHostName, Hostname: ffsEvent.DomainName, User: user, IP: ips, @@ -301,58 +357,51 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg code42Event := &eventOutput.Code42Event{ Id: ffsEvent.EventId, Type: ffsEvent.EventType, - Timestamp: ffsEvent.EventTimestamp, + Timestamp: &eventTimestamp, } code42File := &eventOutput.Code42File{ - Path: ffsEvent.FilePath, - Name: ffsEvent.FileName, - Type: ffsEvent.FileType, - Category: ffsEvent.FileCategory, - IdentifiedExtensionCategory: ffsEvent.IdentifiedExtensionCategory, - CurrentExtensionCategory: ffsEvent.CurrentExtensionCategory, - Size: ffsEvent.FileSize, - Owner: ffsEvent.FileOwner, - Hash: hash, - CreatedTimestamp: ffsEvent.CreatedTimestamp, - ModifyTimestamp: ffsEvent.ModifyTimestamp, - Id: ffsEvent.FileId, - IdentifiedExtensionMIMEType: ffsEvent.IdentifiedExtensionMIMEType, - CurrentExtensionMIMEType: ffsEvent.CurrentExtensionMIMEType, - SuspiciousFileTypeMismatch: ffsEvent.SuspiciousFileTypeMismatch, + Path: ffsEvent.FilePath, + Name: ffsEvent.FileName, + Type: ffsEvent.FileType, + Category: ffsEvent.FileCategory, + MimeTypeByBytes: ffsEvent.MimeTypeByBytes, + MimeTypeByExtension: ffsEvent.MimeTypeByExtension, + Size: ffsEvent.FileSize, + Owner: ffsEvent.FileOwner, + Hash: hash, + CreateTimestamp: &createTimestamp, + ModifyTimestamp: &modifyTimestamp, + Id: ffsEvent.FileId, + MimeTypeMismatch: ffsEvent.MimeTypeMismatch, } code42Device := &eventOutput.Code42Device{ - Username: ffsEvent.DeviceUsername, + Username: ffsEvent.DeviceUserName, Uid: ffsEvent.DeviceUid, } - //code 42 tab fields - var tabURLs []eventOutput.URL + //tabs to tabs + var tabs []eventOutput.Code42TabTab + if ffsEvent.Tabs != nil && len(ffsEvent.Tabs) > 0 { + for _, tab := range ffsEvent.Tabs { + code42Tab := eventOutput.Code42TabTab{ + Title: tab.Title, + Url: getUrlInfo(tab.Url), + } - if ffsEvent.TabURLs != nil && len(ffsEvent.TabURLs) != 0 { - for _, tabUrl := range ffsEvent.TabURLs { - tabURLs = append(tabURLs, *getUrlInfo(tabUrl)) + tabs = append(tabs, code42Tab) } - } - - if tabURLs != nil && len(tabURLs) == 0 { - tabURLs = nil - } - - code42Tab := &eventOutput.Code42Tab{ - WindowTitle: ffsEvent.TabWindowTitle, - Url: getUrlInfo(ffsEvent.TabUrl), - Titles: ffsEvent.TabTitles, - Urls: tabURLs, + } else { + tabs = nil } code42 := &eventOutput.Code42{ Event: code42Event, - InsertionTimestamp: ffsEvent.InsertionTimestamp, + InsertionTimestamp: &insertionTimestamp, File: code42File, Device: code42Device, - OsHostName: ffsEvent.OsHostname, + OsHostName: ffsEvent.OsHostName, DomainName: ffsEvent.DomainName, PublicIpAddress: ffsEvent.PublicIpAddress, PrivateIpAddresses: ffsEvent.PrivateIpAddresses, @@ -367,7 +416,6 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg DetectionSourceAlias: ffsEvent.DetectionSourceAlias, Exposure: ffsEvent.Exposure, Process: process, - Tab: code42Tab, RemovableMedia: &eventOutput.Code42RemovableMedia{ Vendor: ffsEvent.RemovableMediaVendor, Name: ffsEvent.RemovableMediaName, @@ -381,30 +429,30 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg SyncDestination: ffsEvent.SyncDestination, SyncDestinationUsername: ffsEvent.SyncDestinationUsername, EmailDlp: &eventOutput.Code42EmailDlp{ - PolicyNames: ffsEvent.EmailDLPPolicyNames, - Subject: ffsEvent.EmailDLPSubject, - Sender: ffsEvent.EmailDLPSender, - From: ffsEvent.EmailDLPFrom, - Recipients: ffsEvent.EmailDLPRecipients, + PolicyNames: ffsEvent.EmailDlpPolicyNames, + Subject: ffsEvent.EmailSubject, + Sender: ffsEvent.EmailSender, + From: ffsEvent.EmailFrom, + Recipients: ffsEvent.EmailRecipients, }, OutsideActiveHours: ffsEvent.OutsideActiveHours, Print: &eventOutput.Code42Print{ - JobName: ffsEvent.PrintJobName, - PrinterName: ffsEvent.PrinterName, - PrintedFilesBackupPath: ffsEvent.PrintedFilesBackupPath, + JobName: ffsEvent.PrintJobName, + PrinterName: ffsEvent.PrinterName, }, - RemoteActivity: ffsEvent.RemoteActivity, - Trusted: ffsEvent.Trusted, - LoggedInOperatingSystemUser: ffsEvent.LoggedInOperatingSystemUser, + RemoteActivity: ffsEvent.RemoteActivity, + Trusted: ffsEvent.Trusted, + OperatingSystemUser: ffsEvent.OperatingSystemUser, Destination: &eventOutput.Code42Destination{ Category: ffsEvent.DestinationCategory, Name: ffsEvent.DetectionSourceAlias, }, + Tabs: tabs, } elasticFileEvent := &eventOutput.ElasticFileEvent{ Event: event, - Timestamp: timestamp, + Timestamp: ×tamp, File: file, Host: host, Code42: code42, @@ -514,10 +562,18 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg go func() { for _, ffsEvent := range ffsEvents { var indexTime time.Time + + //eventTimestamp + eventTimestamp, err := time.Parse("2006-01-02 15:04:05.000", strings.Replace(strings.Replace(ffsEvent.EventTimestamp, "T", " ", -1), "Z", "", -1)) + + if err != nil { + panic(err) + } + if query.Elasticsearch.IndexTimeGen == "insertionTimestamp" { - indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, ffsEvent.EventTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) + indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, eventTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) } else { - indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, ffsEvent.EventTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) + indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, eventTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) } requiredIndexMutex.RLock() @@ -599,9 +655,23 @@ func queryFetcher(query config.FFSQuery, inProgressQueries *[]eventOutput.InProg for _, ffsEvent := range ffsEvents { var indexTime time.Time if query.Elasticsearch.IndexTimeGen == "insertionTimestamp" { - indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, ffsEvent.InsertionTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) + //eventTimestamp + insertionTimestamp, err := time.Parse("2006-01-02 15:04:05.000", strings.Replace(strings.Replace(ffsEvent.InsertionTimestamp, "T", " ", -1), "Z", "", -1)) + + if err != nil { + panic(err) + } + + indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, insertionTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) } else { - indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, ffsEvent.EventTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) + //eventTimestamp + eventTimestamp, err := time.Parse("2006-01-02 15:04:05.000", strings.Replace(strings.Replace(ffsEvent.EventTimestamp, "T", " ", -1), "Z", "", -1)) + + if err != nil { + panic(err) + } + + indexTime, _ = time.Parse(query.Elasticsearch.IndexTimeAppend, eventTimestamp.Format(query.Elasticsearch.IndexTimeAppend)) } indexName := elasticsearch.BuildIndexNameWithTime(query.Elasticsearch, indexTime) r := elastic.NewBulkIndexRequest().Index(indexName).Doc(ffsEvent) diff --git a/go.mod b/go.mod index 2f9f91a..ef0b244 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/BenB196/crashplan-ffs-puller go 1.15 require ( - github.com/BenB196/crashplan-ffs-go-pkg v0.3.1 + github.com/BenB196/crashplan-ffs-go-pkg v0.4.7 github.com/BenB196/ip-api-go-pkg v0.0.9 github.com/VictoriaMetrics/fastcache v1.5.7 github.com/olivere/elastic/v7 v7.0.22 diff --git a/go.sum b/go.sum index 166eaeb..31100f2 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,22 @@ github.com/BenB196/crashplan-ffs-go-pkg v0.3.0 h1:6daarZ2qt8Wf7g/6uMCc+PVGI/C14W github.com/BenB196/crashplan-ffs-go-pkg v0.3.0/go.mod h1:xyuyYD7cF11Qf/4lPhVcuyd/Rj+BD/ZF710qgdb7KxE= github.com/BenB196/crashplan-ffs-go-pkg v0.3.1 h1:jCB30+j7VqqA0ep3MaFL1BXpck8vA0wiI7A4srujmy0= github.com/BenB196/crashplan-ffs-go-pkg v0.3.1/go.mod h1:xyuyYD7cF11Qf/4lPhVcuyd/Rj+BD/ZF710qgdb7KxE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.0 h1:KwnKzJrEEhJafIhr7oHfYJeZ+tUzn0ubRMaIGvxViy0= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.0/go.mod h1:xyuyYD7cF11Qf/4lPhVcuyd/Rj+BD/ZF710qgdb7KxE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.1 h1:TwLn+DClrVnTC64PXyHeBuPEg0OLnf+aGnGVwv22HAk= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.1/go.mod h1:xyuyYD7cF11Qf/4lPhVcuyd/Rj+BD/ZF710qgdb7KxE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.2 h1:+OqV21+6X5rdu3SvQ8bCyCEjgZ55TDkt/Dm7//c99UE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.2/go.mod h1:xyuyYD7cF11Qf/4lPhVcuyd/Rj+BD/ZF710qgdb7KxE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.3 h1:W++d5WPUqCIFuXJf62rFM9tkByAtoIaHGdfi27p/yPE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.3/go.mod h1:xyuyYD7cF11Qf/4lPhVcuyd/Rj+BD/ZF710qgdb7KxE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.4 h1:NV3YnXWiZYP6m+pDDXtUuaaSsv/X+qeqKY7+uCAX5iA= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.4/go.mod h1:xyuyYD7cF11Qf/4lPhVcuyd/Rj+BD/ZF710qgdb7KxE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.5 h1:DYGG5apCae3bzzsIIH1URyGQAvtHxGXkzO4XWlR+szE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.5/go.mod h1:xyuyYD7cF11Qf/4lPhVcuyd/Rj+BD/ZF710qgdb7KxE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.6 h1:loARC0iPy3Ae3rDb3dGBBJ9RalBw0urCgkEv24ZsppE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.6/go.mod h1:xyuyYD7cF11Qf/4lPhVcuyd/Rj+BD/ZF710qgdb7KxE= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.7 h1:l58BmP+7JpnUYlvBLPbB/olkvOOD2yE2QW1XEud+K18= +github.com/BenB196/crashplan-ffs-go-pkg v0.4.7/go.mod h1:xyuyYD7cF11Qf/4lPhVcuyd/Rj+BD/ZF710qgdb7KxE= github.com/BenB196/ip-api-go-pkg v0.0.3 h1:FWmM7FkhT1N55jd4jPW7W9LVOQrG89DLlrgwMb5fosw= github.com/BenB196/ip-api-go-pkg v0.0.3/go.mod h1:ccPdkBNnzf/uvuk7qXgEO06TCS/qILNJQP/KETQG4jU= github.com/BenB196/ip-api-go-pkg v0.0.4 h1:BbWELxooG6l2gaXQ4i4gm6NsyikdbhalF+TKg08gDPQ=