From c31083901fb7f25794faad5d7da43af94f3aa2d0 Mon Sep 17 00:00:00 2001 From: Brian Ginsburg <7957636+bgins@users.noreply.github.com> Date: Fri, 31 Jan 2025 12:51:57 -0800 Subject: [PATCH] feat: Add matching sorts (#497) * test: Separate match test cases into a testing file * test: Add match offers benchmark * test: Add benchmark GH action * docs: Add benchmarking docs * feat: Add GetResourceOffers query oldest first * feat: Add GetJobOffers query oldest first * fix: Return match result values instead of refs * test: Lift test store helpers to solver package * feat: Matcher retrieve offers oldest first * feat: Add isCheaperOrOlder matching helper * test: Add TestIsCheaperOrOlder unit test * feat: Add pricing and age sorting preferences * test: Add matching and sorting integration test We integrate at the solver to test matcher and store functionality together. * feat: Add service logger to matcher * chore: Add lookup and match decision add error logs * refactor: Move adding match decisions into a helper function * chore: Log matched resource offer decisions count --- .github/workflows/bench.yml | 58 ++++ LOCAL_DEVELOPMENT.md | 15 + README.md | 13 +- pkg/solver/controller.go | 2 +- pkg/solver/matcher/match.go | 28 +- pkg/solver/matcher/matcher.go | 162 +++++++---- pkg/solver/matcher/matcher_bench_test.go | 19 ++ pkg/solver/matcher/matcher_test.go | 345 ++++------------------- pkg/solver/matcher/testing.go | 334 ++++++++++++++++++++++ pkg/solver/solver_test.go | 152 ++++++++++ pkg/solver/store/db/db.go | 6 + pkg/solver/store/memory/store.go | 19 ++ pkg/solver/store/store.go | 6 + pkg/solver/store/store_test.go | 319 ++++++++++----------- pkg/solver/testing.go | 127 +++++++++ stack | 9 +- 16 files changed, 1093 insertions(+), 521 deletions(-) create mode 100644 .github/workflows/bench.yml create mode 100644 pkg/solver/matcher/matcher_bench_test.go create mode 100644 pkg/solver/matcher/testing.go create mode 100644 pkg/solver/solver_test.go create mode 100644 pkg/solver/testing.go diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 00000000..49eed9fe --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,58 @@ +name: Benchmarks + +on: + pull_request: + branches: + - main + push: + branches: + - main + +permissions: + deployments: write + contents: write + +jobs: + run-solver-benchmarks: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Install golang + uses: actions/setup-go@v5 + + - name: Run benchmarks + run: ./stack benchmarks-solver | tee assets/benchmark-output.txt + + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + + - name: Report benchmark results + if: github.event_name == 'pull_request' + uses: benchmark-action/github-action-benchmark@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + tool: 'go' + output-file-path: assets/benchmark-output.txt + external-data-json-path: ./cache/benchmark-data.json + # Comment when performance degrades >2x + comment-on-alert: true + # Add summary to action run + summary-always: true + auto-push: false + + - name: Publish benchmark results + if: github.event_name == 'push' && github.ref_name == 'main' + uses: benchmark-action/github-action-benchmark@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + tool: 'go' + output-file-path: assets/benchmark-output.txt + # Add summary to action run + summary-always: true + # Publish gh-pages chart + auto-push: true diff --git a/LOCAL_DEVELOPMENT.md b/LOCAL_DEVELOPMENT.md index 6672f3b4..ea17155d 100644 --- a/LOCAL_DEVELOPMENT.md +++ b/LOCAL_DEVELOPMENT.md @@ -97,6 +97,21 @@ Run the Go unit tests with `./stack unit-tests` and the Hardhat unit tests with Run the integration tests with `./stack integration-tests`. The integration tests expect all parts of the stack are running, except the request to run a job. See [Using Docker Compose](./LOCAL_DEVELOPMENT.md#using-docker-compose) to run the stack. +#### Benchmarks + +We perform relative benchmarking on our solver matching algorithm to avoid performance degradation. The results over time are displayed in charts here: https://lilypad-tech.github.io/lilypad/dev/bench/. + +Each CI benchmark test run displays results in the GitHub action summary for the previous and current commit. An [example run](https://github.com/Lilypad-Tech/lilypad/actions/runs/12942901815) shows four measurements that are also reflected in the charts: + +- BenchmarkMatchOffers: A summary of overall measurements +- BenchmarkMatchOffers - ns/op: Nanoseconds per operation +- BenchmarkMatchOffers - B/op: Number of bytes allocated per operation +- BenchmarkMatchOffers - allocs/op: Number of allocations per operation + +Each op is a run of the matching algorithm over a set of job and resource offers designed to exercise the possible matching decisions. + +When the benchmark detects a 2x or greater performance degradation, it displays a commit comment with a perforance alert. See this [example commit](https://github.com/Lilypad-Tech/lilypad/commit/249cca9fe568ecfc6f04813b48ad46ccc0c76258) that intentionally introduces a performance degradation. + ## Notes on tooling Things should work right out-of-the-box, no extra configuration should be needed as Doppler provides the environment variables that are required with the current setup. diff --git a/README.md b/README.md index 21d25b3e..d9311896 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -LIKE THIS PROJECT? +LIKE THIS PROJECT? PLEASE STAR US AND HELP US GROW! <3 @@ -18,7 +18,7 @@ lilypad run cowsay:v0.0.4 -i Message="moo" [![Watch the video](https://img.youtube.com/vi/Ep9ML9h8DTE/0.jpg)](https://www.youtube.com/watch?v=Ep9ML9h8DTE) -The current list of modules can be found in the following repositories: +The current list of modules can be found in the following repositories: * [lilysay](https://github.com/Lilypad-Tech/lilypad-module-lilysay) @@ -31,9 +31,13 @@ Containerised job modules can be built and added to the available module list; f ## Getting started running a Node on Lilypad Network -As a distributed network Lilypad also brings with it the ability to run as a node and contribute to the GPU and compute capabilities. See the documentation on [running a node](https://lilypad.team/node) which contains more details instructions and overview for getting set up. +As a distributed network Lilypad also brings with it the ability to run as a node and contribute to the GPU and compute capabilities. See the documentation on [running a node](https://lilypad.team/node) which contains more details instructions and overview for getting set up. -## The Lilypad Community +## Performance + +We benchmark our solver job matching algorithm: https://lilypad-tech.github.io/lilypad/dev/bench/. See our [benchmark guide](./LOCAL_DEVELOPMENT.md#benchmarks) for more details. + +## The Lilypad Community [Read our Blog](https://lilypad.team/blog) @@ -42,4 +46,3 @@ As a distributed network Lilypad also brings with it the ability to run as a nod [Follow us on Twitter/X](https://lilypad.team/x) [Check out our videos on YouTube](https://lilypad.team/youtube) - diff --git a/pkg/solver/controller.go b/pkg/solver/controller.go index 314da7a3..c5aea5c8 100644 --- a/pkg/solver/controller.go +++ b/pkg/solver/controller.go @@ -283,7 +283,7 @@ func (controller *SolverController) solve(ctx context.Context) error { defer span.End() // find out which deals we can make from matching the offers - deals, err := matcher.GetMatchingDeals(ctx, controller.store, controller.updateJobOfferState, controller.tracer, controller.meter) + deals, err := matcher.GetMatchingDeals(ctx, controller.store, controller.updateJobOfferState, controller.log, controller.tracer, controller.meter) if err != nil { span.SetStatus(codes.Error, "get matching deals failed") span.RecordError(err) diff --git a/pkg/solver/matcher/match.go b/pkg/solver/matcher/match.go index 3ff04c15..bfef899f 100644 --- a/pkg/solver/matcher/match.go +++ b/pkg/solver/matcher/match.go @@ -211,6 +211,7 @@ func (result priceMismatch) attributes() []attribute.KeyValue { } } +// TODO(bgins) Rename to validator type mediatorMismatch struct { resourceOffer data.ResourceOffer jobOffer data.JobOffer @@ -253,19 +254,19 @@ func matchOffers( jobOffer data.JobOffer, ) matchResult { if resourceOffer.Spec.CPU < jobOffer.Spec.CPU { - return &cpuMismatch{ + return cpuMismatch{ jobOffer: jobOffer, resourceOffer: resourceOffer, } } if resourceOffer.Spec.GPU < jobOffer.Spec.GPU { - return &gpuMismatch{ + return gpuMismatch{ jobOffer: jobOffer, resourceOffer: resourceOffer, } } if resourceOffer.Spec.RAM < jobOffer.Spec.RAM { - return &ramMismatch{ + return ramMismatch{ jobOffer: jobOffer, resourceOffer: resourceOffer, } @@ -275,7 +276,7 @@ func matchOffers( if len(jobOffer.Spec.GPUs) > 0 { // Mismatch if job offer requests VRAM but resource provider has none if len(resourceOffer.Spec.GPUs) == 0 { - return &vramMismatch{ + return vramMismatch{ jobOffer: jobOffer, resourceOffer: resourceOffer, } @@ -285,7 +286,7 @@ func matchOffers( largestResourceOfferVRAM := getLargestVRAM(resourceOffer.Spec.GPUs) largestJobOfferVRAM := getLargestVRAM(jobOffer.Spec.GPUs) if largestResourceOfferVRAM < largestJobOfferVRAM { - return &vramMismatch{ + return vramMismatch{ jobOffer: jobOffer, resourceOffer: resourceOffer, } @@ -293,7 +294,7 @@ func matchOffers( } if resourceOffer.Spec.Disk < jobOffer.Spec.Disk { - return &diskSpaceMismatch{ + return diskSpaceMismatch{ jobOffer: jobOffer, resourceOffer: resourceOffer, } @@ -301,7 +302,7 @@ func matchOffers( moduleID, err := data.GetModuleID(jobOffer.Module) if err != nil { - return &moduleIDError{ + return moduleIDError{ jobOffer: jobOffer, resourceOffer: resourceOffer, err: err, @@ -320,7 +321,7 @@ func matchOffers( } if !hasModule { - return &moduleMismatch{ + return moduleMismatch{ jobOffer: jobOffer, resourceOffer: resourceOffer, moduleID: moduleID, @@ -330,7 +331,7 @@ func matchOffers( // we don't currently support market priced resource offers if resourceOffer.Mode == data.MarketPrice { - return &marketPriceUnavailable{ + return marketPriceUnavailable{ resourceOffer: resourceOffer, } } @@ -338,7 +339,7 @@ func matchOffers( // if both are fixed price then we filter out "cannot afford" if resourceOffer.Mode == data.FixedPrice && jobOffer.Mode == data.FixedPrice { if resourceOffer.DefaultPricing.InstructionPrice > jobOffer.Pricing.InstructionPrice { - return &priceMismatch{ + return priceMismatch{ jobOffer: jobOffer, resourceOffer: resourceOffer, moduleID: moduleID, @@ -346,22 +347,23 @@ func matchOffers( } } + // TODO(bgins) Rename to validator mutualMediators := data.GetMutualServices(resourceOffer.Services.Mediator, jobOffer.Services.Mediator) if len(mutualMediators) == 0 { - return &mediatorMismatch{ + return mediatorMismatch{ jobOffer: jobOffer, resourceOffer: resourceOffer, } } if resourceOffer.Services.Solver != jobOffer.Services.Solver { - return &solverMismatch{ + return solverMismatch{ jobOffer: jobOffer, resourceOffer: resourceOffer, } } - return &offersMatched{ + return offersMatched{ jobOffer: jobOffer, resourceOffer: resourceOffer, } diff --git a/pkg/solver/matcher/matcher.go b/pkg/solver/matcher/matcher.go index 726cf79b..7c92a823 100644 --- a/pkg/solver/matcher/matcher.go +++ b/pkg/solver/matcher/matcher.go @@ -3,30 +3,24 @@ package matcher import ( "context" "errors" - "sort" + "fmt" "github.com/lilypad-tech/lilypad/pkg/data" "github.com/lilypad-tech/lilypad/pkg/solver/store" "github.com/lilypad-tech/lilypad/pkg/system" "github.com/rs/zerolog/log" + zerolog "github.com/rs/zerolog/log" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" ) -type ListOfResourceOffers []data.ResourceOffer - -func (a ListOfResourceOffers) Len() int { return len(a) } -func (a ListOfResourceOffers) Less(i, j int) bool { - return a[i].DefaultPricing.InstructionPrice < a[j].DefaultPricing.InstructionPrice -} -func (a ListOfResourceOffers) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - func GetMatchingDeals( ctx context.Context, db store.SolverStore, updateJobOfferState func(string, string, uint8) (*data.JobOfferContainer, error), + log *system.ServiceLogger, tracer trace.Tracer, meter metric.Meter, ) ([]data.Deal, error) { @@ -36,12 +30,14 @@ func GetMatchingDeals( deals := []data.Deal{} - // Get resource offers + // Get resource offers oldest first span.AddEvent("db.get_resource_offers.start") resourceOffers, err := db.GetResourceOffers(store.GetResourceOffersQuery{ - NotMatched: true, + NotMatched: true, + OrderOldestFirst: true, }) if err != nil { + log.Error("get resource offers failed", err) span.SetStatus(codes.Error, "get resource offers failed") span.RecordError(err) return nil, err @@ -52,12 +48,14 @@ func GetMatchingDeals( }) span.AddEvent("db.get_resource_offers.done") - // Get job offers + // Get job offers oldest first span.AddEvent("db.get_job_offers.start") jobOffers, err := db.GetJobOffers(store.GetJobOffersQuery{ - NotMatched: true, + NotMatched: true, + OrderOldestFirst: true, }) if err != nil { + log.Error("get job offers failed", err) span.SetStatus(codes.Error, "get job offers failed") span.RecordError(err) return nil, err @@ -68,9 +66,8 @@ func GetMatchingDeals( }) span.AddEvent("db.get_resource_offers.done") - // loop over job offers + // Loop over job offers attempting to match with resource offers for _, jobOffer := range jobOffers { - // Check for targeted jobs if jobOffer.JobOffer.Target.Address != "" { deal, err := getTargetedDeal(ctx, db, jobOffer, updateJobOfferState, tracer) @@ -84,7 +81,7 @@ func GetMatchingDeals( continue } - // loop over resource offers + // Build list of matching resource offers matchingResourceOffers := []data.ResourceOffer{} for _, resourceOffer := range resourceOffers { _, matchSpan := tracer.Start(ctx, "match", @@ -101,7 +98,7 @@ func GetMatchingDeals( } matchSpan.AddEvent("db.get_match_decision.done") - // if this exists it means we've already tried to match the two elements and should not try again + // If a decision exists it means we've already tried to match the offers and should not try again if decision != nil { matchSpan.AddEvent("decision_already_checked", trace.WithAttributes(attribute.Bool("decision.result", decision.Result))) @@ -109,12 +106,14 @@ func GetMatchingDeals( continue } + // Check for a match matchSpan.AddEvent("match_offers.start") result := matchOffers(resourceOffer.ResourceOffer, jobOffer.JobOffer) logMatch(result) matchSpan.AddEvent("match_offers.done", trace.WithAttributes(result.attributes()...)) if result.matched() { + // Match found, add it to the matched list matchingResourceOffers = append(matchingResourceOffers, resourceOffer.ResourceOffer) matchSpan.AddEvent("append_match", trace.WithAttributes(attribute.KeyValue{ @@ -122,6 +121,7 @@ func GetMatchingDeals( Value: attribute.StringSliceValue(data.GetResourceOfferIDs(matchingResourceOffers)), })) } else { + // No match, add a decision without a deal ID and a false result matchSpan.AddEvent("add_match_decision.start") _, err := db.AddMatchDecision(resourceOffer.ID, jobOffer.ID, "", false) if err != nil { @@ -136,49 +136,66 @@ func GetMatchingDeals( } // yay - we've got some matching resource offers - // let's choose the cheapest one + // Now let's match the by pricing and age preference. if len(matchingResourceOffers) > 0 { - // now let's order the matching resource offers by price - sort.Sort(ListOfResourceOffers(matchingResourceOffers)) - cheapestResourceOffer := matchingResourceOffers[0] - - span.AddEvent("get_deal.start", trace.WithAttributes(attribute.String("cheapest_resource_offer", cheapestResourceOffer.ID), - attribute.KeyValue{ - Key: "matching_resource_offers", - Value: attribute.StringSliceValue(data.GetResourceOfferIDs(matchingResourceOffers)), - })) - deal, err := data.GetDeal(jobOffer.JobOffer, cheapestResourceOffer) - if err != nil { - span.SetStatus(codes.Error, "unable to get deal") - span.RecordError(err) - return nil, err + var selectedResourceOffer data.ResourceOffer + if jobOffer.JobOffer.Mode == data.MarketPrice { + // For market price jobs, select the cheapest offer, breaking ties with age + selectedResourceOffer = matchingResourceOffers[0] + for _, offer := range matchingResourceOffers[1:] { + if isCheaperOrOlder(offer, selectedResourceOffer) { + selectedResourceOffer = offer + } + } + } else { + // For fixed price jobs, select the first (oldest) offer that meets the price requirement + for _, offer := range matchingResourceOffers { + if offer.DefaultPricing.InstructionPrice <= jobOffer.JobOffer.Pricing.InstructionPrice { + selectedResourceOffer = offer + break + } + } } - span.AddEvent("get_deal.done", trace.WithAttributes(attribute.String("deal.id", deal.ID))) - - // add the match decision for this job offer - for _, matchingResourceOffer := range matchingResourceOffers { - addDealID := "" - if cheapestResourceOffer.ID == matchingResourceOffer.ID { - addDealID = deal.ID + // Selected resource offer ID will be set when we selected a resource offer. + // Otherwise it will be a zero-value empty string. + if selectedResourceOffer.ID != "" { + span.AddEvent("get_deal.start", + trace.WithAttributes( + attribute.String("selected_resource_offer", selectedResourceOffer.ID), + attribute.KeyValue{ + Key: "matching_resource_offers", + Value: attribute.StringSliceValue(data.GetResourceOfferIDs(matchingResourceOffers)), + })) + + // Make a deal with the job offer and selected resource offer + deal, err := data.GetDeal(jobOffer.JobOffer, selectedResourceOffer) + if err != nil { + span.SetStatus(codes.Error, "unable to get deal") + span.RecordError(err) + return nil, err } + span.AddEvent("get_deal.done", trace.WithAttributes(attribute.String("deal.id", deal.ID))) - span.AddEvent("add_match_decision.start") - _, err := db.AddMatchDecision(matchingResourceOffer.ID, jobOffer.ID, addDealID, true) + // Add match decisions for all matching offers + span.AddEvent("add_match_decisions.start") + err = addMatchDecisions(db, jobOffer.ID, deal.ID, selectedResourceOffer, matchingResourceOffers) if err != nil { + log.Error("addMatchDecisions failed", err) span.SetStatus(codes.Error, "unable to add match decision") span.RecordError(err) return nil, err } - span.AddEvent("add_match_decision.done") - } + span.AddEvent("add_match_decisions.done") - deals = append(deals, deal) - span.AddEvent("append_deal", - trace.WithAttributes(attribute.KeyValue{ - Key: "deals", - Value: attribute.StringSliceValue(data.GetDealIDs(deals)), - })) + // Add deal to overall deals list + deals = append(deals, deal) + span.AddEvent("append_deal", + trace.WithAttributes(attribute.KeyValue{ + Key: "deals", + Value: attribute.StringSliceValue(data.GetDealIDs(deals)), + })) + } } } @@ -187,15 +204,58 @@ func GetMatchingDeals( metrics.resourceOffers.Record(ctx, int64(len(resourceOffers))) metrics.deals.Record(ctx, int64(len(deals))) - log.Debug(). + zerolog.Debug(). Int("jobOffers", len(jobOffers)). Int("resourceOffers", len(resourceOffers)). Int("deals", len(deals)). - Msgf(system.GetServiceString(system.SolverService, "Solver solving")) + Msg(system.GetServiceString(system.SolverService, "Solver solving")) return deals, nil } +func addMatchDecisions( + db store.SolverStore, + jobOfferID string, + dealID string, + selectedResourceOffer data.ResourceOffer, + matchingResourceOffers []data.ResourceOffer, +) error { + for _, matchingOffer := range matchingResourceOffers { + addDealID := "" + if selectedResourceOffer.ID == matchingOffer.ID { + addDealID = dealID + } + + // All match decisions had matching resource offers, set the result to true. + // The match decision has a deal ID if it's resource offer was selected. + _, err := db.AddMatchDecision(matchingOffer.ID, jobOfferID, addDealID, true) + if err != nil { + return fmt.Errorf( + "unable to add match decision for job offer %s and matched resource offer %s: %s", + jobOfferID, + matchingOffer.ID, + err, + ) + } + } + zerolog.Debug(). + Int("decisions", len(matchingResourceOffers)). + Msg(system.GetServiceString(system.SolverService, "Solver adding matched resource offer decisions")) + + return nil +} + +// Returns true if first offer is cheaper, or has same price but is older +func isCheaperOrOlder(a, b data.ResourceOffer) bool { + priceA := a.DefaultPricing.InstructionPrice + priceB := b.DefaultPricing.InstructionPrice + + if priceA != priceB { + return priceA < priceB // Choose cheaper price + } + return a.CreatedAt < b.CreatedAt // Choose older offer +} + // See if our jobOffer targets a specific address. If so, we will create a deal automatically // with the matcing resourceOffer. func getTargetedDeal( diff --git a/pkg/solver/matcher/matcher_bench_test.go b/pkg/solver/matcher/matcher_bench_test.go new file mode 100644 index 00000000..da10912f --- /dev/null +++ b/pkg/solver/matcher/matcher_bench_test.go @@ -0,0 +1,19 @@ +//go:build bench && solver + +package matcher + +import "testing" + +func BenchmarkMatchOffers(b *testing.B) { + testCases := getMatchTestCases() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tc := range testCases { + result := matchOffers(tc.resourceOffer, tc.jobOffer) + if result.matched() != tc.shouldMatch { + b.Fatalf("expected match to be %v for case %s", tc.shouldMatch, tc.name) + } + } + } +} diff --git a/pkg/solver/matcher/matcher_test.go b/pkg/solver/matcher/matcher_test.go index 8d4599aa..0ab12140 100644 --- a/pkg/solver/matcher/matcher_test.go +++ b/pkg/solver/matcher/matcher_test.go @@ -9,317 +9,80 @@ import ( ) func TestMatchOffers(t *testing.T) { - services := data.ServiceConfig{ - Solver: "oranges", - Mediator: []string{"apples"}, - } - - basicResourceOffer := data.ResourceOffer{ - Spec: data.MachineSpec{ - CPU: 1000, - GPU: 1000, - RAM: 1024, - GPUs: []data.GPUSpec{ - { - Name: "NVIDIA RTX 4090", - Vendor: "NVIDIA", - VRAM: 24576, // MB - }, - }, - Disk: 2924295844659, // Bytes - }, - DefaultPricing: data.DealPricing{ - InstructionPrice: 10, - }, - Mode: data.FixedPrice, - Services: services, - } - - basicJobOffer := data.JobOffer{ - Spec: data.MachineSpec{ - CPU: 1000, - GPU: 1000, - RAM: 1024, - GPUs: []data.GPUSpec{}, - Disk: 0, // Bytes - }, - Mode: data.MarketPrice, - Services: services, - } - - cowsayModuleConfig := data.ModuleConfig{ - Name: "cowsay", - Repo: "https://github.com/Lilypad-Tech/lilypad-module-cowsay", - Hash: "v0.0.4", - Path: "/lilypad_module.json.tmpl", - } - - lilysayModuleConfig := data.ModuleConfig{ - Name: "lilysay", - Repo: "https://github.com/Lilypad-Tech/lilypad-module-lilysay", - Hash: "v0.5.2", - Path: "/lilypad_module.json.tmpl", - } - - sdxlModuleConfig := data.ModuleConfig{ - Name: "", - Repo: "https://github.com/Lilypad-Tech/lilypad-module-sdxl", - Hash: "v0.9-lilypad1", - Path: "/lilypad_module.json.tmpl", + testCases := getMatchTestCases() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := matchOffers(tc.resourceOffer, tc.jobOffer) + if result.matched() != tc.shouldMatch { + t.Errorf("Expected match to be %v, but got %+v", tc.shouldMatch, result) + } + }) } +} +func TestIsCheaperOrOlder(t *testing.T) { testCases := []struct { - name string - resourceOffer func(offer data.ResourceOffer) data.ResourceOffer - jobOffer func(offer data.JobOffer) data.JobOffer - shouldMatch bool + name string + offerA data.ResourceOffer + offerB data.ResourceOffer + expected bool }{ { - name: "Basic match", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - return offer - }, - shouldMatch: true, - }, - { - name: "CPU mismatch", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Spec.CPU = 2000 - return offer - }, - shouldMatch: false, - }, - { - name: "GPU mismatch", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Spec.GPU = 2000 - return offer - }, - shouldMatch: false, - }, - { - name: "RAM mismatch", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Spec.GPU = 2048 - return offer - }, - shouldMatch: false, - }, - { - name: "VRAM match when job creator specifies VRAM", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Spec.GPUs = []data.GPUSpec{{VRAM: 24576}} - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Module = sdxlModuleConfig - offer.Spec.GPUs = []data.GPUSpec{{VRAM: 20000}} - return offer - }, - shouldMatch: true, - }, - { - name: "VRAM match when job creator does not specify VRAM", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Spec.GPUs = []data.GPUSpec{{VRAM: 24576}} - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Module = sdxlModuleConfig - offer.Spec.GPUs = []data.GPUSpec{} - return offer - }, - shouldMatch: true, - }, - { - name: "VRAM requested is more than resource offer VRAM", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Spec.GPUs = []data.GPUSpec{{VRAM: 24576}} - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Module = sdxlModuleConfig - offer.Spec.GPUs = []data.GPUSpec{{VRAM: 40000}} - return offer - }, - shouldMatch: false, - }, - { - name: "VRAM requested but resource offer has none", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Spec.GPUs = []data.GPUSpec{} - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Module = sdxlModuleConfig - offer.Spec.GPUs = []data.GPUSpec{{VRAM: 49152}} - return offer - }, - shouldMatch: false, - }, - { - name: "Disk space match when job creator specifies disk space", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Spec.Disk = 2924295844659 // Bytes - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Module = sdxlModuleConfig - offer.Spec.Disk = 1000000000000 - return offer - }, - shouldMatch: true, - }, - { - name: "Disk space match when job creator does not specify disk space", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Spec.Disk = 2924295844659 // Bytes - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Module = sdxlModuleConfig - offer.Spec.Disk = 0 // zero-value - return offer - }, - shouldMatch: true, - }, - { - name: "Disk space requested is more than resource offer disk space", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Spec.Disk = 2924295844659 // Bytes - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Spec.Disk = 4000000000000 - return offer - }, - shouldMatch: false, - }, - { - name: "Resource provider supports module", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - moduleID, _ := data.GetModuleID(cowsayModuleConfig) - offer.Modules = []string{moduleID} - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Module = cowsayModuleConfig - return offer - }, - shouldMatch: true, - }, - { - name: "Resource provider does not support module", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - moduleID, _ := data.GetModuleID(cowsayModuleConfig) - offer.Modules = []string{moduleID} - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Module = lilysayModuleConfig - return offer - }, - shouldMatch: false, - }, - { - name: "Empty mediators", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Services.Mediator = []string{} - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Services.Mediator = []string{} - return offer - }, - shouldMatch: false, - }, - { - name: "Fixed price - too expensive", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Mode = data.FixedPrice - offer.Pricing.InstructionPrice = 9 - return offer - }, - shouldMatch: false, - }, - { - name: "Fixed price - can afford", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - offer.Mode = data.FixedPrice - offer.Pricing.InstructionPrice = 11 - return offer - }, - shouldMatch: true, - }, - { - name: "Resource provider using unimplemented market pricing", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Mode = data.MarketPrice - return offer - }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - return offer - }, - shouldMatch: false, - }, - { - name: "Mismatched mediators", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Services.Mediator = []string{"apples2"} - return offer + name: "cheaper wins", + offerA: data.ResourceOffer{ + CreatedAt: 2, + DefaultPricing: data.DealPricing{ + InstructionPrice: 100, + }, }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - return offer + offerB: data.ResourceOffer{ + CreatedAt: 1, + DefaultPricing: data.DealPricing{ + InstructionPrice: 200, + }, }, - shouldMatch: false, + expected: true, }, { - name: "Different mediators with one matching mediator", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Services.Mediator = []string{"apples2", "apples"} - return offer + name: "older wins when same price", + offerA: data.ResourceOffer{ + CreatedAt: 1, + DefaultPricing: data.DealPricing{ + InstructionPrice: 100, + }, }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - return offer + offerB: data.ResourceOffer{ + CreatedAt: 2, + DefaultPricing: data.DealPricing{ + InstructionPrice: 100, + }, }, - shouldMatch: true, + expected: true, }, { - name: "Different solver", - resourceOffer: func(offer data.ResourceOffer) data.ResourceOffer { - offer.Services.Solver = "pears" - return offer + name: "more expensive loses regardless of age", + offerA: data.ResourceOffer{ + CreatedAt: 1, + DefaultPricing: data.DealPricing{ + InstructionPrice: 200, + }, }, - jobOffer: func(offer data.JobOffer) data.JobOffer { - return offer + offerB: data.ResourceOffer{ + CreatedAt: 2, + DefaultPricing: data.DealPricing{ + InstructionPrice: 100, + }, }, - shouldMatch: false, + expected: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - result := matchOffers(tc.resourceOffer(basicResourceOffer), tc.jobOffer(basicJobOffer)) - if result.matched() != tc.shouldMatch { - t.Errorf("Expected match to be %v, but got %+v", tc.shouldMatch, result) + result := isCheaperOrOlder(tc.offerA, tc.offerB) + if result != tc.expected { + t.Errorf("isCheaperOrOlder(%+v, %+v) = %v, expected %v", + tc.offerA, tc.offerB, result, tc.expected) } }) } diff --git a/pkg/solver/matcher/testing.go b/pkg/solver/matcher/testing.go new file mode 100644 index 00000000..5c06511b --- /dev/null +++ b/pkg/solver/matcher/testing.go @@ -0,0 +1,334 @@ +//go:build unit || bench + +package matcher + +import "github.com/lilypad-tech/lilypad/pkg/data" + +type matchTestCase struct { + name string + resourceOffer data.ResourceOffer + jobOffer data.JobOffer + expectedResult string + shouldMatch bool +} + +func getMatchTestCases() []matchTestCase { + // Base configurations + baseServices := data.ServiceConfig{ + Solver: "oranges", + Mediator: []string{"apples"}, + } + + baseResourceOffer := data.ResourceOffer{ + Spec: data.MachineSpec{ + CPU: 1000, + GPU: 1000, + RAM: 1024, + Disk: 2924295844659, + GPUs: []data.GPUSpec{{ + Name: "NVIDIA RTX 4090", + Vendor: "NVIDIA", + VRAM: 24576, + }}, + }, + DefaultPricing: data.DealPricing{ + InstructionPrice: 10, + }, + Mode: data.FixedPrice, + Services: baseServices, + } + + baseJobOffer := data.JobOffer{ + Spec: data.MachineSpec{ + CPU: 1000, + GPU: 1000, + RAM: 1024, + Disk: 2924295844659, + GPUs: []data.GPUSpec{}, + }, + Mode: data.MarketPrice, + Services: baseServices, + } + + // Module configs + cowsayModuleConfig := data.ModuleConfig{ + Name: "cowsay", + Repo: "https://github.com/Lilypad-Tech/lilypad-module-cowsay", + Hash: "v0.0.4", + Path: "/lilypad_module.json.tmpl", + } + + lilysayModuleConfig := data.ModuleConfig{ + Name: "lilysay", + Repo: "https://github.com/Lilypad-Tech/lilypad-module-lilysay", + Hash: "v0.5.2", + Path: "/lilypad_module.json.tmpl", + } + + testCases := []matchTestCase{ + // Matching cases + { + name: "Basic match", + resourceOffer: baseResourceOffer, + jobOffer: baseJobOffer, + expectedResult: "*matcher.offersMatched", + shouldMatch: true, + }, + { + name: "VRAM match when job creator specifies VRAM", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Spec.GPUs = []data.GPUSpec{{VRAM: 24576}} + return r + }(), + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Spec.GPUs = []data.GPUSpec{{VRAM: 20000}} + return j + }(), + expectedResult: "*matcher.offersMatched", + shouldMatch: true, + }, + { + name: "VRAM match when job creator does not specify VRAM", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Spec.GPUs = []data.GPUSpec{{VRAM: 24576}} + return r + }(), + jobOffer: func() data.JobOffer { + j := baseJobOffer + return j + }(), + expectedResult: "*matcher.offersMatched", + shouldMatch: true, + }, + { + name: "Disk space match when job creator specifies disk space", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Spec.Disk = 2924295844659 // Bytes + return r + }(), + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Spec.Disk = 1000000000000 + return j + }(), + expectedResult: "*matcher.offersMatched", + shouldMatch: true, + }, + { + name: "Disk space match when job creator does not specify disk space", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Spec.Disk = 2924295844659 // Bytes + return r + }(), + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Spec.Disk = 0 // zero-value + return j + }(), + expectedResult: "*matcher.offersMatched", + shouldMatch: true, + }, + { + name: "Resource provider supports module", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + moduleID, _ := data.GetModuleID(cowsayModuleConfig) + r.Modules = []string{moduleID} + return r + }(), + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Module = cowsayModuleConfig + return j + }(), + expectedResult: "*matcher.offersMatched", + shouldMatch: true, + }, + { + name: "Fixed price can afford", + resourceOffer: baseResourceOffer, + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Mode = data.FixedPrice + j.Pricing.InstructionPrice = 11 + return j + }(), + expectedResult: "*matcher.offersMatched", + shouldMatch: true, + }, + { + name: "Different mediators with one matching mediator", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Services.Mediator = []string{"apples2", "apples"} + return r + }(), + jobOffer: baseJobOffer, + expectedResult: "*matcher.offersMatched", + shouldMatch: true, + }, + + // Mismatch cases + { + name: "CPU mismatch", + resourceOffer: baseResourceOffer, + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Spec.CPU = 2000 + return j + }(), + expectedResult: "*matcher.cpuMismatch", + shouldMatch: false, + }, + { + name: "GPU mismatch", + resourceOffer: baseResourceOffer, + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Spec.GPU = 2000 + return j + }(), + expectedResult: "*matcher.gpuMismatch", + shouldMatch: false, + }, + { + name: "RAM mismatch", + resourceOffer: baseResourceOffer, + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Spec.RAM = 2048 + return j + }(), + expectedResult: "*matcher.ramMismatch", + shouldMatch: false, + }, + { + name: "VRAM requested is more than resource offer VRAM", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Spec.GPUs = []data.GPUSpec{{VRAM: 24576}} + return r + }(), + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Spec.GPUs = []data.GPUSpec{{VRAM: 40000}} + return j + }(), + expectedResult: "*matcher.vramMismatch", + shouldMatch: false, + }, + { + name: "VRAM requested but resource offer has none", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Spec.GPUs = []data.GPUSpec{} + return r + }(), + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Spec.GPUs = []data.GPUSpec{{VRAM: 8192}} + return j + }(), + expectedResult: "*matcher.vramMismatch", + shouldMatch: false, + }, + { + name: "Disk space requested is more than resource offer disk space", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Spec.Disk = 2924295844659 // Bytes + return r + }(), + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Spec.Disk = 4000000000000 + return j + }(), + expectedResult: "*matcher.diskSpaceMismatch", + shouldMatch: false, + }, + { + name: "Resource provider does not support module", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + moduleID, _ := data.GetModuleID(cowsayModuleConfig) + r.Modules = []string{moduleID} + return r + }(), + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Module = lilysayModuleConfig + return j + }(), + expectedResult: "*matcher.moduleMismatch", + shouldMatch: false, + }, + { + name: "Resource provider using unimplemented market pricing", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Mode = data.MarketPrice + return r + }(), + jobOffer: baseJobOffer, + expectedResult: "*matcher.marketPriceUnavailable", + shouldMatch: false, + }, + { + name: "Fixed price too expensive", + resourceOffer: baseResourceOffer, + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Mode = data.FixedPrice + j.Pricing.InstructionPrice = 9 + return j + }(), + expectedResult: "*matcher.priceMismatch", + shouldMatch: false, + }, + { + name: "Mismatched mediators", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Services.Mediator = []string{"apples2"} + return r + }(), + jobOffer: baseJobOffer, + expectedResult: "*matcher.mediatorMismatch", + shouldMatch: false, + }, + { + name: "Empty mediators", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Services.Mediator = []string{} + return r + }(), + jobOffer: func() data.JobOffer { + j := baseJobOffer + j.Services.Mediator = []string{} + return j + }(), + expectedResult: "*matcher.mediatorMismatch", + shouldMatch: false, + }, + { + name: "Mismatched solver", + resourceOffer: func() data.ResourceOffer { + r := baseResourceOffer + r.Services.Solver = "pears" + return r + }(), + jobOffer: baseJobOffer, + expectedResult: "*matcher.solverMismatch", + shouldMatch: false, + }, + } + + return testCases +} diff --git a/pkg/solver/solver_test.go b/pkg/solver/solver_test.go new file mode 100644 index 00000000..81747882 --- /dev/null +++ b/pkg/solver/solver_test.go @@ -0,0 +1,152 @@ +//go:build integration && solver + +package solver_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/lilypad-tech/lilypad/pkg/data" + "github.com/lilypad-tech/lilypad/pkg/solver" + "github.com/lilypad-tech/lilypad/pkg/solver/matcher" + "github.com/lilypad-tech/lilypad/pkg/system" + "go.opentelemetry.io/otel" +) + +func TestMatchingSortingLogic(t *testing.T) { + testCases := []struct { + name string + jobOffer data.JobOfferContainer + resourceOffers []data.ResourceOfferContainer + expectedWinner string + }{ + { + name: "market price - selects cheapest and oldest offer", + jobOffer: getJobOffer("job-market", data.MarketPrice, 0), + resourceOffers: []data.ResourceOfferContainer{ + getResourceOffer("cheap-old", 1, 100), + getResourceOffer("expensive-new", 2, 200), + getResourceOffer("cheap-newer", 3, 100), + }, + expectedWinner: "cheap-old", + }, + { + name: "market price - prefers cheaper over older", + jobOffer: getJobOffer("job-market", data.MarketPrice, 0), + resourceOffers: []data.ResourceOfferContainer{ + getResourceOffer("expensive-old", 1, 200), + getResourceOffer("cheap-new", 2, 100), + }, + expectedWinner: "cheap-new", + }, + { + name: "fixed price - selects oldest affordable offer", + jobOffer: getJobOffer("job-fixed", data.FixedPrice, 150), + resourceOffers: []data.ResourceOfferContainer{ + getResourceOffer("cheap-old", 1, 100), + getResourceOffer("expensive-new", 2, 200), + getResourceOffer("cheap-newer", 3, 100), + }, + expectedWinner: "cheap-old", + }, + { + name: "fixed price - exact price match prefers oldest", + jobOffer: getJobOffer("job-fixed", data.FixedPrice, 100), + resourceOffers: []data.ResourceOfferContainer{ + getResourceOffer("exact-old", 1, 100), + getResourceOffer("exact-newer", 2, 100), + getResourceOffer("exact-newest", 3, 100), + }, + expectedWinner: "exact-old", + }, + } + + storeConfigs := solver.SetupTestStores(t) + for _, config := range storeConfigs { + getStore, clearStore := config.Init() + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%s/%s", config.Name, tc.name), func(t *testing.T) { + store := getStore() + defer clearStore() + + // Add job offer + _, err := store.AddJobOffer(tc.jobOffer) + if err != nil { + t.Fatalf("Failed to add job offer: %v", err) + } + + // The database store implementation sorts by database timestamp, + // so we insert with a small delay to create an ordering. The memory implementation + // sorts by resource offer created time, so we assign one in our test cases. + for i, offer := range tc.resourceOffers { + _, err := store.AddResourceOffer(offer) + if err != nil { + t.Fatalf("Failed to add resource offer: %v", err) + } + + if i < len(tc.resourceOffers)-1 { + time.Sleep(50 * time.Millisecond) + } + } + + // Run matcher + deals, err := matcher.GetMatchingDeals( + context.Background(), + store, + func(string, string, uint8) (*data.JobOfferContainer, error) { return nil, nil }, + system.NewServiceLogger("solver"), + otel.Tracer("test"), + otel.Meter("test"), + ) + if err != nil { + t.Fatalf("GetMatchingDeals failed: %v", err) + } + + if len(deals) != 1 { + t.Fatalf("Expected 1 deal, got %d", len(deals)) + } + if deals[0].ResourceOffer.ID != tc.expectedWinner { + t.Errorf("Expected resource offer %s to win, got %s", + tc.expectedWinner, deals[0].ResourceOffer.ID) + } + }) + } + } +} + +func getJobOffer(id string, mode data.PricingMode, maxPrice uint64) data.JobOfferContainer { + return data.JobOfferContainer{ + ID: id, + JobOffer: data.JobOffer{ + ID: id, + Mode: mode, + Pricing: data.DealPricing{ + InstructionPrice: maxPrice, + }, + Services: data.ServiceConfig{ + Solver: "default-solver", + Mediator: []string{"mediator1"}, + }, + }, + } +} + +func getResourceOffer(id string, createdAt int, price uint64) data.ResourceOfferContainer { + return data.ResourceOfferContainer{ + ID: id, + ResourceOffer: data.ResourceOffer{ + ID: id, + CreatedAt: createdAt, + DefaultPricing: data.DealPricing{ + InstructionPrice: price, + }, + Services: data.ServiceConfig{ + Solver: "default-solver", + Mediator: []string{"mediator1"}, + }, + }, + } +} diff --git a/pkg/solver/store/db/db.go b/pkg/solver/store/db/db.go index 9d8aab99..36a33e45 100644 --- a/pkg/solver/store/db/db.go +++ b/pkg/solver/store/db/db.go @@ -148,6 +148,9 @@ func (store *SolverStoreDatabase) GetJobOffers(query store.GetJobOffersQuery) ([ if !query.IncludeCancelled { q = q.Where("state != ?", data.GetAgreementStateIndex("JobOfferCancelled")) } + if query.OrderOldestFirst { + q = q.Order("created_at ASC") + } var records []JobOffer if err := q.Find(&records).Error; err != nil { @@ -178,6 +181,9 @@ func (store *SolverStoreDatabase) GetResourceOffers(query store.GetResourceOffer data.GetAgreementStateIndex("DealAgreed"), }) } + if query.OrderOldestFirst { + q = q.Order("created_at ASC") + } var records []ResourceOffer if err := q.Find(&records).Error; err != nil { diff --git a/pkg/solver/store/memory/store.go b/pkg/solver/store/memory/store.go index bc83ce3b..04274a73 100644 --- a/pkg/solver/store/memory/store.go +++ b/pkg/solver/store/memory/store.go @@ -2,6 +2,7 @@ package store import ( "fmt" + "sort" "strings" "sync" @@ -82,6 +83,7 @@ func (s *SolverStoreMemory) AddMatchDecision(resourceOffer string, jobOffer stri func (s *SolverStoreMemory) GetJobOffers(query store.GetJobOffersQuery) ([]data.JobOfferContainer, error) { s.mutex.RLock() defer s.mutex.RUnlock() + jobOffers := []data.JobOfferContainer{} for _, jobOffer := range s.jobOfferMap { matching := true @@ -100,12 +102,21 @@ func (s *SolverStoreMemory) GetJobOffers(query store.GetJobOffersQuery) ([]data. jobOffers = append(jobOffers, *jobOffer) } } + + // Sort oldest first + if query.OrderOldestFirst { + sort.Slice(jobOffers, func(i, j int) bool { + return jobOffers[i].JobOffer.CreatedAt < jobOffers[j].JobOffer.CreatedAt + }) + } + return jobOffers, nil } func (s *SolverStoreMemory) GetResourceOffers(query store.GetResourceOffersQuery) ([]data.ResourceOfferContainer, error) { s.mutex.RLock() defer s.mutex.RUnlock() + resourceOffers := []data.ResourceOfferContainer{} for _, resourceOffer := range s.resourceOfferMap { matching := true @@ -124,6 +135,14 @@ func (s *SolverStoreMemory) GetResourceOffers(query store.GetResourceOffersQuery resourceOffers = append(resourceOffers, *resourceOffer) } } + + // Sort oldest first + if query.OrderOldestFirst { + sort.Slice(resourceOffers, func(i, j int) bool { + return resourceOffers[i].ResourceOffer.CreatedAt < resourceOffers[j].ResourceOffer.CreatedAt + }) + } + return resourceOffers, nil } diff --git a/pkg/solver/store/store.go b/pkg/solver/store/store.go index 61f86045..42fb7e73 100644 --- a/pkg/solver/store/store.go +++ b/pkg/solver/store/store.go @@ -22,6 +22,9 @@ type GetJobOffersQuery struct { // this will include cancelled job offers in the results IncludeCancelled bool `json:"include_cancelled"` + + // Sort job offers oldest first + OrderOldestFirst bool `json:"order_oldest_first"` } type GetResourceOffersQuery struct { @@ -44,6 +47,9 @@ type GetResourceOffersQuery struct { // we use the DealID property of the resourceOfferContainer to tell if it's been matched NotMatched bool `json:"not_matched"` + + // Sort resource offers oldest first + OrderOldestFirst bool `json:"order_oldest_first"` } type GetDealsQuery struct { diff --git a/pkg/solver/store/store_test.go b/pkg/solver/store/store_test.go index e06bdaea..d2b9124b 100644 --- a/pkg/solver/store/store_test.go +++ b/pkg/solver/store/store_test.go @@ -8,25 +8,23 @@ import ( "sort" "sync" "testing" + "time" "github.com/lilypad-tech/lilypad/pkg/data" + "github.com/lilypad-tech/lilypad/pkg/solver" "github.com/lilypad-tech/lilypad/pkg/solver/store" solverstore "github.com/lilypad-tech/lilypad/pkg/solver/store" - databasestore "github.com/lilypad-tech/lilypad/pkg/solver/store/db" - memorystore "github.com/lilypad-tech/lilypad/pkg/solver/store/memory" "golang.org/x/exp/rand" ) -const DB_CONN_STR = "postgres://postgres:postgres@localhost:5432/solver-db?sslmode=disable" - // Job offers func TestJobOfferOps(t *testing.T) { - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { // Test multiple job offers in a single test run - t.Run(config.name, func(t *testing.T) { - getStore, clearStore := config.init() + t.Run(config.Name, func(t *testing.T) { + getStore, clearStore := config.Init() store := getStore() defer clearStore() @@ -220,14 +218,14 @@ func TestJobOfferQuery(t *testing.T) { }, } - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { - getStore, clearStore := config.init() + getStore, clearStore := config.Init() defer clearStore() for _, tc := range testCases { // Test each case in a separate test run - t.Run(fmt.Sprintf("%s/%s", config.name, tc.name), func(t *testing.T) { + t.Run(fmt.Sprintf("%s/%s", config.Name, tc.name), func(t *testing.T) { store := getStore() // Add test job offers @@ -262,14 +260,67 @@ func TestJobOfferQuery(t *testing.T) { } } +func TestJobOfferSort(t *testing.T) { + const delay = 50 * time.Millisecond + numOffers := 3 + rand.Intn(5) // Random offers between 3 and 7 + + storeConfigs := solver.SetupTestStores(t) + for _, config := range storeConfigs { + getStore, clearStore := config.Init() + defer clearStore() + + t.Run(config.Name, func(t *testing.T) { + store := getStore() + + // Track IDs in insertion order + var insertedIDs []string + + // The database GetJobOffers implementation sorts by database timestamp, + // so we insert with a small delay to create an ordering. The memory implementation + // sorts by job offer created time, so we assign one. + for i := 0; i < numOffers; i++ { + offer := generateJobOfferWithTime(i) + _, err := store.AddJobOffer(offer) + if err != nil { + t.Fatalf("Failed to add job offer: %v", err) + } + insertedIDs = append(insertedIDs, offer.ID) + + if i < numOffers-1 { + time.Sleep(delay) + } + } + + // Run query sorting oldest offer first + results, err := store.GetJobOffers(solverstore.GetJobOffersQuery{ + OrderOldestFirst: true, + }) + if err != nil { + t.Fatalf("GetJobOffers failed: %v", err) + } + + // Extract IDs from results + resultIDs := make([]string, len(results)) + for i, result := range results { + resultIDs[i] = result.ID + } + + // Expect same order as insertion (oldest first) + if !slices.Equal(resultIDs, insertedIDs) { + t.Errorf("Expected order %v, got %v", insertedIDs, resultIDs) + } + }) + } +} + // Resource Offer func TestResourceOfferOps(t *testing.T) { - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { // Test multiple resource offers in a single test run - t.Run(config.name, func(t *testing.T) { - getStore, clearStore := config.init() + t.Run(config.Name, func(t *testing.T) { + getStore, clearStore := config.Init() store := getStore() defer clearStore() @@ -475,14 +526,14 @@ func TestResourceOfferQuery(t *testing.T) { }, } - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { - getStore, clearStore := config.init() + getStore, clearStore := config.Init() defer clearStore() for _, tc := range testCases { // Test each case in a separate test run - t.Run(fmt.Sprintf("%s/%s", config.name, tc.name), func(t *testing.T) { + t.Run(fmt.Sprintf("%s/%s", config.Name, tc.name), func(t *testing.T) { store := getStore() // Add test resource offers @@ -517,14 +568,67 @@ func TestResourceOfferQuery(t *testing.T) { } } +func TestResourceOfferSort(t *testing.T) { + const delay = 50 * time.Millisecond + numOffers := 3 + rand.Intn(5) // Random offers between 3 and 7 + + storeConfigs := solver.SetupTestStores(t) + for _, config := range storeConfigs { + getStore, clearStore := config.Init() + defer clearStore() + + t.Run(config.Name, func(t *testing.T) { + store := getStore() + + // Track IDs in insertion order + var insertedIDs []string + + // The database GetResourceOffers implementation sorts by database timestamp, + // so we insert with a small delay to create an ordering. The memory implementation + // sorts by resource offer created time, so we assign one. + for i := 0; i < numOffers; i++ { + offer := generateResourceOfferWithTime(i) + _, err := store.AddResourceOffer(offer) + if err != nil { + t.Fatalf("Failed to add resource offer: %v", err) + } + insertedIDs = append(insertedIDs, offer.ID) + + if i < numOffers-1 { + time.Sleep(delay) + } + } + + // Run query sorting oldest offer first + results, err := store.GetResourceOffers(solverstore.GetResourceOffersQuery{ + OrderOldestFirst: true, + }) + if err != nil { + t.Fatalf("GetResourceOffers failed: %v", err) + } + + // Extract IDs from results + resultIDs := make([]string, len(results)) + for i, result := range results { + resultIDs[i] = result.ID + } + + // Expect same order as insertion (oldest first) + if !slices.Equal(resultIDs, insertedIDs) { + t.Errorf("Expected order %v, got %v", insertedIDs, resultIDs) + } + }) + } +} + // Deals func TestDealOps(t *testing.T) { - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { // Test multiple deals in a single test run - t.Run(config.name, func(t *testing.T) { - getStore, clearStore := config.init() + t.Run(config.Name, func(t *testing.T) { + getStore, clearStore := config.Init() store := getStore() defer clearStore() @@ -573,11 +677,11 @@ func TestDealOps(t *testing.T) { } func TestDealGetAll(t *testing.T) { - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { // Test batch of deals in a test run - t.Run(config.name, func(t *testing.T) { - getStore, clearStore := config.init() + t.Run(config.Name, func(t *testing.T) { + getStore, clearStore := config.Init() store := getStore() defer clearStore() @@ -624,11 +728,11 @@ func TestDealGetAll(t *testing.T) { } func TestDealUpdates(t *testing.T) { - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { // Test multiple deals in a single test run - t.Run(config.name, func(t *testing.T) { - getStore, clearStore := config.init() + t.Run(config.Name, func(t *testing.T) { + getStore, clearStore := config.Init() store := getStore() defer clearStore() @@ -828,14 +932,14 @@ func TestDealQuery(t *testing.T) { }, } - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { - getStore, clearStore := config.init() + getStore, clearStore := config.Init() defer clearStore() for _, tc := range testCases { // Test each case in a separate test run - t.Run(fmt.Sprintf("%s/%s", config.name, tc.name), func(t *testing.T) { + t.Run(fmt.Sprintf("%s/%s", config.Name, tc.name), func(t *testing.T) { store := getStore() // Add deals @@ -873,11 +977,11 @@ func TestDealQuery(t *testing.T) { // Results func TestResultOps(t *testing.T) { - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { // Test multiple results in a single test run - t.Run(config.name, func(t *testing.T) { - getStore, clearStore := config.init() + t.Run(config.Name, func(t *testing.T) { + getStore, clearStore := config.Init() store := getStore() defer clearStore() @@ -972,11 +1076,11 @@ func TestResultOps(t *testing.T) { // Match decisions func TestMatchDecisionOps(t *testing.T) { - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { // Test multiple match decisions in a single test run - t.Run(config.name, func(t *testing.T) { - getStore, clearStore := config.init() + t.Run(config.Name, func(t *testing.T) { + getStore, clearStore := config.Init() store := getStore() defer clearStore() @@ -1095,10 +1199,10 @@ func TestMatchDecisionOps(t *testing.T) { } func TestMatchDecisionRemove(t *testing.T) { - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { - t.Run(config.name, func(t *testing.T) { - getStore, clearStore := config.init() + t.Run(config.Name, func(t *testing.T) { + getStore, clearStore := config.Init() store := getStore() defer clearStore() @@ -1162,11 +1266,11 @@ func TestConcurrentOps(t *testing.T) { results := generateResults(4, 10) matchDecisions := generateMatchDecisions(4, 10) - storeConfigs := setupStores(t) + storeConfigs := solver.SetupTestStores(t) for _, config := range storeConfigs { // Test concurrent adds in a single test run - t.Run(config.name, func(t *testing.T) { - getStore, clearStore := config.init() + t.Run(config.Name, func(t *testing.T) { + getStore, clearStore := config.Init() store := getStore() defer clearStore() @@ -1340,124 +1444,6 @@ func TestConcurrentOps(t *testing.T) { } } -// Utilities - -type storeConfig struct { - name string - init func() (getStore func() store.SolverStore, clearStore func()) -} - -func setupStores(t *testing.T) []storeConfig { - initMemory := func() (func() store.SolverStore, func()) { - // Get store function creates a new memory store - // which effectively clears data between runs - getStore := func() store.SolverStore { - s, err := memorystore.NewSolverStoreMemory() - if err != nil { - t.Fatalf("Failed to create memory store: %v", err) - } - return s - } - clearStore := func() {} - - return getStore, clearStore - } - - initDatabase := func() (func() store.SolverStore, func()) { - db, err := databasestore.NewSolverStoreDatabase(DB_CONN_STR, "silent") - if err != nil { - t.Fatalf("Failed to create database store: %v", err) - } - - // Clear data at initialization - clearStoreDatabase(t, db) - - // Get store functions clear data and returns - // the same store instance - getStore := func() store.SolverStore { - clearStoreDatabase(t, db) - return db - } - clearStore := func() { clearStoreDatabase(t, db) } - - return getStore, clearStore - } - - return []storeConfig{ - {name: "memory", init: initMemory}, - {name: "database", init: initDatabase}, - } -} - -func clearStoreDatabase(t *testing.T, s store.SolverStore) { - // Delete job offers - jobOffers, err := s.GetJobOffers(store.GetJobOffersQuery{ - IncludeCancelled: true, - }) - if err != nil { - t.Fatalf("Failed to get existing job offers: %v", err) - } - - for _, result := range jobOffers { - err := s.RemoveJobOffer(result.ID) - if err != nil { - t.Fatalf("Failed to remove existing job offer: %v", err) - } - } - - // Delete resource offers - resourceOffers, err := s.GetResourceOffers(store.GetResourceOffersQuery{}) - if err != nil { - t.Fatalf("Failed to get existing resource offers: %v", err) - } - - for _, result := range resourceOffers { - err := s.RemoveResourceOffer(result.ID) - if err != nil { - t.Fatalf("Failed to remove existing resource offer: %v", err) - } - } - - // Delete deals - deals, err := s.GetDealsAll() - if err != nil { - t.Fatalf("Failed to get existing deals: %v", err) - } - - for _, deal := range deals { - err := s.RemoveDeal(deal.ID) - if err != nil { - t.Fatalf("Failed to remove existing deal: %v", err) - } - } - - // Delete results - results, err := s.GetResults() - if err != nil { - t.Fatalf("Failed to get existing results: %v", err) - } - - for _, result := range results { - err := s.RemoveResult(result.DealID) - if err != nil { - t.Fatalf("Failed to remove existing result: %v", err) - } - } - - // Delete match decisions - decisions, err := s.GetMatchDecisions() - if err != nil { - t.Fatalf("Failed to get existing match decisions: %v", err) - } - - for _, decision := range decisions { - err := s.RemoveMatchDecision(decision.ResourceOffer, decision.JobOffer) - if err != nil { - t.Fatalf("Failed to remove existing match decision: %v", err) - } - } -} - // Generators func generateCID() string { @@ -1489,8 +1475,16 @@ func generateState() uint8 { } func generateJobOffer() data.JobOfferContainer { + return generateJobOfferWithTime(rand.Intn(1000000)) +} + +func generateJobOfferWithTime(createdAt int) data.JobOfferContainer { return data.JobOfferContainer{ - ID: generateCID(), + ID: generateCID(), + JobCreator: generateEthAddress(), + JobOffer: data.JobOffer{ + CreatedAt: createdAt, + }, } } @@ -1506,9 +1500,16 @@ func generateJobOffers(min int, max int) []data.JobOfferContainer { } func generateResourceOffer() data.ResourceOfferContainer { + return generateResourceOfferWithTime(rand.Intn(1000000)) +} + +func generateResourceOfferWithTime(createdAt int) data.ResourceOfferContainer { return data.ResourceOfferContainer{ ID: generateCID(), ResourceProvider: generateEthAddress(), + ResourceOffer: data.ResourceOffer{ + CreatedAt: createdAt, + }, } } diff --git a/pkg/solver/testing.go b/pkg/solver/testing.go new file mode 100644 index 00000000..52690689 --- /dev/null +++ b/pkg/solver/testing.go @@ -0,0 +1,127 @@ +package solver + +import ( + "testing" + + "github.com/lilypad-tech/lilypad/pkg/solver/store" + databasestore "github.com/lilypad-tech/lilypad/pkg/solver/store/db" + memorystore "github.com/lilypad-tech/lilypad/pkg/solver/store/memory" +) + +const DB_CONN_STR = "postgres://postgres:postgres@localhost:5432/solver-db?sslmode=disable" + +type TestStoreConfig struct { + Name string + Init func() (getStore func() store.SolverStore, clearStore func()) +} + +func SetupTestStores(t *testing.T) []TestStoreConfig { + initMemory := func() (func() store.SolverStore, func()) { + // Get store function creates a new memory store + // which effectively clears data between runs + getStore := func() store.SolverStore { + s, err := memorystore.NewSolverStoreMemory() + if err != nil { + t.Fatalf("Failed to create memory store: %v", err) + } + return s + } + clearStore := func() {} + + return getStore, clearStore + } + + initDatabase := func() (func() store.SolverStore, func()) { + db, err := databasestore.NewSolverStoreDatabase(DB_CONN_STR, "silent") + if err != nil { + t.Fatalf("Failed to create database store: %v", err) + } + + // Clear data at initialization + clearStoreDatabase(t, db) + + // Get store functions clear data and returns + // the same store instance + getStore := func() store.SolverStore { + clearStoreDatabase(t, db) + return db + } + clearStore := func() { clearStoreDatabase(t, db) } + + return getStore, clearStore + } + + return []TestStoreConfig{ + {Name: "memory", Init: initMemory}, + {Name: "database", Init: initDatabase}, + } +} + +func clearStoreDatabase(t *testing.T, s store.SolverStore) { + // Delete job offers + jobOffers, err := s.GetJobOffers(store.GetJobOffersQuery{ + IncludeCancelled: true, + }) + if err != nil { + t.Fatalf("Failed to get existing job offers: %v", err) + } + + for _, result := range jobOffers { + err := s.RemoveJobOffer(result.ID) + if err != nil { + t.Fatalf("Failed to remove existing job offer: %v", err) + } + } + + // Delete resource offers + resourceOffers, err := s.GetResourceOffers(store.GetResourceOffersQuery{}) + if err != nil { + t.Fatalf("Failed to get existing resource offers: %v", err) + } + + for _, result := range resourceOffers { + err := s.RemoveResourceOffer(result.ID) + if err != nil { + t.Fatalf("Failed to remove existing resource offer: %v", err) + } + } + + // Delete deals + deals, err := s.GetDealsAll() + if err != nil { + t.Fatalf("Failed to get existing deals: %v", err) + } + + for _, deal := range deals { + err := s.RemoveDeal(deal.ID) + if err != nil { + t.Fatalf("Failed to remove existing deal: %v", err) + } + } + + // Delete results + results, err := s.GetResults() + if err != nil { + t.Fatalf("Failed to get existing results: %v", err) + } + + for _, result := range results { + err := s.RemoveResult(result.DealID) + if err != nil { + t.Fatalf("Failed to remove existing result: %v", err) + } + } + + // Delete match decisions + decisions, err := s.GetMatchDecisions() + if err != nil { + t.Fatalf("Failed to get existing match decisions: %v", err) + } + + for _, decision := range decisions { + err := s.RemoveMatchDecision(decision.ResourceOffer, decision.JobOffer) + if err != nil { + t.Fatalf("Failed to remove existing match decision: %v", err) + } + } +} diff --git a/stack b/stack index f44bfe39..b7e64632 100755 --- a/stack +++ b/stack @@ -268,8 +268,15 @@ function integration-tests() { # Base services and the solver must be running function integration-tests-solver() { load-local-env - go test -v -tags="integration,solver" -count 1 ./... + # Run integration tests for each package serially to isolate database interactions + go test -tags="integration,solver" ./.../solver -v -count 1 + go test -tags="integration,solver" ./.../store -v -count 1 } + +function benchmarks-solver() { + go test -tags="bench,solver" -bench=MatchOffers -benchtime=3s -benchmem ./... +} + ############################################################################ # run #