From 0f7740c04e96b93090067e0d34fe1e2bf01adc27 Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Wed, 20 Nov 2024 01:55:06 +0000 Subject: [PATCH] Add comments and final tests --- .../query/util/MetadataHelperCacheTest.java | 240 ++++++++++++------ 1 file changed, 166 insertions(+), 74 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index d609f5d75b..c42c092379 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -13,6 +13,7 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -20,12 +21,10 @@ import java.util.Set; import java.util.concurrent.ExecutionException; -import datawave.IdentityDataType; -import datawave.data.type.DateType; -import datawave.data.type.GeoType; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.SecurityOperations; @@ -50,6 +49,9 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import datawave.IdentityDataType; +import datawave.data.type.DateType; +import datawave.data.type.GeoType; import datawave.query.composite.CompositeMetadataHelper; // enable springification @@ -262,17 +264,8 @@ public void setup() { assertNotNull(alternateAuthsHelper); assertNotNull(alternateHelper); assertNotNull(cacheManager); - - // TODO delete this later - // expect(accumuloClient.instanceOperations()).andAnswer(new IAnswer() { - // @Override - // public InstanceOperations answer() throws Throwable { - // return null; - // } - // }).anyTimes(); } - // wrapper so I can track access while testing private AccumuloClient getAccumuloClient() { return accumuloClient; } @@ -894,11 +887,11 @@ public void expect() throws TableNotFoundException { // @formatter:on } - // validate key = "{#root.target.metadataTableName}" - private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations() { - return getNoArgMetadataTableNameVariations(null); - } - + /** + * Create expectations for no argument method calls for each of the metadataHelper objects. Verify a second call returns the same result as the first without additional accumulo calls. When extra is set, also expect this on the initial call + * @param extra + * @return + */ private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations(ExtraExpectation extra) { MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {}) { public void expect() throws Exception { @@ -1080,6 +1073,11 @@ public void expect() throws TableNotFoundException { } private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations(Args[] args, boolean argsInKey, boolean exactMatch, Expectation alwaysExpect) { + return getAuthMetadataTableNameVariations(args, argsInKey, exactMatch, alwaysExpect, 1); + } + + private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations(Args[] args, boolean argsInKey, boolean exactMatch, Expectation alwaysExpect, + int callsPerCache) { if (args == null) { args = new Args[] {new Args(new Object[] {})}; } @@ -1096,9 +1094,11 @@ private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations(Args[] if (first || argsInKey) { base = new MethodParamsAndExpectations(metadataHelper, a.args) { public void expect() throws Exception { - Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); - expectScanner("table", s, entries); + for (int i = 0; i < callsPerCache; i++) { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + } if (alwaysExpect != null) { alwaysExpect.expect(); @@ -1108,9 +1108,11 @@ public void expect() throws Exception { altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, a.args) { public void expect() throws Exception { - Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); - expectScanner("table", s, entries); + for (int i = 0; i < callsPerCache; i++) { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + } if (alwaysExpect != null) { alwaysExpect.expect(); @@ -1120,9 +1122,11 @@ public void expect() throws Exception { altTable = new MethodParamsAndExpectations(alternateTableMetadataHelper, a.args) { public void expect() throws Exception { - Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); - expectScanner("table2", s, entries); + for (int i = 0; i < callsPerCache; i++) { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + } if (alwaysExpect != null) { alwaysExpect.expect(); @@ -1132,9 +1136,11 @@ public void expect() throws Exception { altTableAndAuths = new MethodParamsAndExpectations(alternateHelper, a.args) { public void expect() throws Exception { - Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); - expectScanner("table2", s, entries); + for (int i = 0; i < callsPerCache; i++) { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + } if (alwaysExpect != null) { alwaysExpect.expect(); @@ -1219,14 +1225,21 @@ private MethodParamsAndExpectations[] getNonEventFieldExpectations(Args[] args) } private MethodParamsAndExpectations[] getScansPerCall(Args[] args, int scansPerTable) { + return getScansPerCall(args, scansPerTable, null); + } + + private MethodParamsAndExpectations[] getScansPerCall(Args[] args, int scansPerTable, ExtraExpectation extra) { List results = new ArrayList<>(); for (int i = 0; i < args.length; i++) { - MethodParamsAndExpectations base = getWithExpectation(metadataHelper, args[i].args, null, false, new ScannerExpectation("table", scansPerTable)); + MethodParamsAndExpectations base = getWithExpectation(metadataHelper, args[i].args, null, false, + new ScannerExpectation("table", scansPerTable, extra)); MethodParamsAndExpectations baseAltTable = getWithExpectation(alternateTableMetadataHelper, args[i].args, null, false, - new ScannerExpectation("table2", scansPerTable)); - MethodParamsAndExpectations baseAltAuths = getWithExpectation(alternateAuthsHelper, args[i].args, null, false, new ScannerExpectation("table", scansPerTable)); - MethodParamsAndExpectations baseAlt = getWithExpectation(alternateHelper, args[i].args, null, false, new ScannerExpectation("table2", scansPerTable)); + new ScannerExpectation("table2", scansPerTable, extra)); + MethodParamsAndExpectations baseAltAuths = getWithExpectation(alternateAuthsHelper, args[i].args, null, false, + new ScannerExpectation("table", scansPerTable, extra)); + MethodParamsAndExpectations baseAlt = getWithExpectation(alternateHelper, args[i].args, null, false, + new ScannerExpectation("table2", scansPerTable, extra)); // no cache hits across any variation here results.add(base); @@ -1235,16 +1248,17 @@ private MethodParamsAndExpectations[] getScansPerCall(Args[] args, int scansPerT results.add(baseAlt); // no caching is happening so should be exactly the same for any follow up calls - results.add(getWithExpectation(metadataHelper, args[i].args, base, false, new ScannerExpectation("table", scansPerTable))); - results.add(getWithExpectation(alternateTableMetadataHelper, args[i].args, baseAltTable, false, new ScannerExpectation("table2", scansPerTable))); - results.add(getWithExpectation(alternateAuthsHelper, args[i].args, baseAltAuths, false, new ScannerExpectation("table", scansPerTable))); - results.add(getWithExpectation(alternateHelper, args[i].args, baseAlt, false, new ScannerExpectation("table2", scansPerTable))); + results.add(getWithExpectation(metadataHelper, args[i].args, base, false, new ScannerExpectation("table", scansPerTable, extra))); + results.add(getWithExpectation(alternateTableMetadataHelper, args[i].args, baseAltTable, false, + new ScannerExpectation("table2", scansPerTable, extra))); + results.add(getWithExpectation(alternateAuthsHelper, args[i].args, baseAltAuths, false, new ScannerExpectation("table", scansPerTable, extra))); + results.add(getWithExpectation(alternateHelper, args[i].args, baseAlt, false, new ScannerExpectation("table2", scansPerTable, extra))); } return results.toArray(new MethodParamsAndExpectations[0]); } - // test all public, non-static caching methods for consistency + // test all public, non-static caching methods for cache consistency. Ensure caching is defined and works as expected for all methods. This test will fail if cache keys are wrong leading to unexpected accumulo calls @Test public void cachingConsistencyTest() throws Exception { // @formatter:off @@ -1316,8 +1330,6 @@ public void cachingConsistencyTest() throws Exception { methodToParams.put("2.getDatatypesForField", getAuthMetadataTableNameVariations(new Args[] { new Args(new Object[] {"f1", Collections.emptySet()}), new Args(new Object[] {"f1", Collections.singleton("filter")}), - // TODO this case should be verified since its a partial key -// new Args(new Object[] {"f2", Collections.singleton("filter")})}, true, false)); new Args(new Object[] {"f2", Collections.singleton("filter2")})}, true, false)); methodToParams.put("0.getTypeMetadata", getNoArgAuthMetadataTableNameVariations(null)); methodToParams.put("1.getTypeMetadata", getAuthMetadataTableNameVariations(new Args[] { @@ -1333,7 +1345,7 @@ public void cachingConsistencyTest() throws Exception { new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, true, true)); - // TODO verifying filters would require even more test setup to mock the type metadata that matches the classes being passed + // verifying filters would require even more test setup to mock the type metadata that matches the classes being passed methodToParams.put("2.getFieldsForDatatype", getAuthMetadataTableNameVariations(new Args[] { new Args(new Object[] {IdentityDataType.class, Collections.emptySet()}), new Args(new Object[] {DateType.class, Collections.emptySet()}), @@ -1363,10 +1375,43 @@ public void cachingConsistencyTest() throws Exception { new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); methodToParams.put("0.loadIndexOnlyFields", getNoArgAuthMetadataTableNameVariations(null)); - // TODO special cases - methodToParams.put("2.getQueryModel", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getQueryModel", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getQueryModel", new MethodParamsAndExpectations[] {}); + // 1 scanner + getAllFields + getIndexOnlyFields caching + methodToParams.put("2.getQueryModel", getAuthMetadataTableNameVariations(new Args[]{ + new Args(new Object[] {"table", "model"}), + new Args(new Object[] {"table", "model2"}), + new Args(new Object[] {"table", "model3"})}, false, false, new ScannerExpectation("table", 1), 2)); + // 1 scanner + getAllFields caching + methodToParams.put("3.getQueryModel", getAuthMetadataTableNameVariations(new Args[]{ + new Args(new Object[] {"table", "model", Collections.emptySet()}), + new Args(new Object[] {"table", "model", Collections.singleton("f1")}), + new Args(new Object[] {"table", "model2", Collections.singleton("f1")}), + new Args(new Object[] {"table", "model2", Collections.singleton("f2")}), + new Args(new Object[] {"table", "model3", Collections.singleton("f1")}), + new Args(new Object[] {"t3", "model3", Collections.singleton("f1")})}, false, false, new ScannerExpectation(null, 1))); + // 1 scanner + getAllFields caching + methodToParams.put("4.getQueryModel", getAuthMetadataTableNameVariations(new Args[]{ + new Args(new Object[] {"table", "model", Collections.emptySet(), Collections.emptySet()}), + new Args(new Object[] {"table", "model", Collections.singleton("f1"), Collections.emptySet()}), + new Args(new Object[] {"table", "model2", Collections.singleton("f1"), Collections.singleton("filter")}), + new Args(new Object[] {"table", "model2", Collections.singleton("f2"), Collections.singleton("filter")}), + new Args(new Object[] {"table", "model3", Collections.singleton("f1"), Collections.singleton("filter")}), + new Args(new Object[] {"t3", "model3", Collections.singleton("f1"), Collections.singleton("filter")})}, false, false, new ScannerExpectation(null, 1))); + // loadIndexedFields, but loadIndexedFields is call from within AllFieldsMetadataHelper so is NOT cached + methodToParams.put("3.getFieldIndexHoles", getScansPerCall(new Args[]{ + // empty set of fields does an extra scan (always) so can't be tested in this test +// new Args(new Object[] {Collections.emptySet(), Collections.emptySet(), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.emptySet(), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.singleton("filter"), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.singleton("filter2"), 3}), + new Args(new Object[] {Collections.singleton("f2"), Collections.singleton("filter2"), 3})}, 1)); + // loadIndexedFields, but loadIndexedFields is call from within AllFieldsMetadataHelper so is NOT cached + methodToParams.put("3.getReversedFieldIndexHoles", getScansPerCall(new Args[]{ + // empty set of fields does an extra scan (always) so can't be tested in this test +// new Args(new Object[] {Collections.emptySet(), Collections.emptySet(), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.emptySet(), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.singleton("filter"), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.singleton("filter2"), 3}), + new Args(new Object[] {Collections.singleton("f2"), Collections.singleton("filter2"), 3})}, 1)); // uncached methods that can safely be skipped methodToParams.put("0.toString", new MethodParamsAndExpectations[] {}); @@ -1400,21 +1445,42 @@ public void cachingConsistencyTest() throws Exception { new Args(new Object[] {})}, 1)); methodToParams.put("0.loadAllFields", getScansPerCall(new Args[] { new Args(new Object[] {})}, 1)); - // TODO from here down for scan verification - methodToParams.put("3.getCardinalityForField", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getCardinalityForField", new MethodParamsAndExpectations[] {}); - methodToParams.put("2.getCountsByFieldInDay", new MethodParamsAndExpectations[] {}); - methodToParams.put("1.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); - methodToParams.put("2.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getCountsForFieldsInDateRange", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getCountsForFieldsInDateRange", new MethodParamsAndExpectations[] {}); - methodToParams.put("1.getEarliestOccurrenceOfField", new MethodParamsAndExpectations[] {}); - methodToParams.put("2.getEarliestOccurrenceOfFieldWithType", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getEarliestOccurrenceOfFieldWithType", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getFieldIndexHoles", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getReversedFieldIndexHoles", new MethodParamsAndExpectations[] {}); + methodToParams.put("3.getCardinalityForField", getScansPerCall(new Args[] { + new Args(new Object[] {"f1", new Date(), new Date()})}, 1)); + methodToParams.put("4.getCardinalityForField", getScansPerCall(new Args[] { + new Args(new Object[] {"f1", "type1", new Date(), new Date()})}, 1)); + methodToParams.put("2.getCountsByFieldInDay", getScansPerCall(new Args[] { + new Args(new Object[] {"f1", "20241119"}), + new Args(new Object[] {"f2", "20241119"}), + new Args(new Object[] {"f1", "20241118"})}, 1, (s)-> { + s.addScanIterator(anyObject()); + })); + methodToParams.put("3.getCountsByFieldInDayWithTypes", getScansPerCall(new Args[] { + new Args(new Object[] {"f1", "20241119", Collections.emptySet()}), + new Args(new Object[] {"f2", "20241119", Collections.emptySet()}), + new Args(new Object[] {"f1", "20241119", Collections.singleton("filter")})}, 1, (s)-> { + s.addScanIterator(anyObject()); + })); + // These use batch scanners... + methodToParams.put("3.getCountsForFieldsInDateRange", getScansPerCall(new Args[] { + new Args(new Object[] {Collections.emptySet(), new Date(), new Date()})}, 0, (s)-> { + BatchScanner bs = createMock(BatchScanner.class); + expect(getAccumuloClient().createBatchScanner(anyObject(), anyObject())).andReturn(bs); + bs.close(); + })); + // can't do this one, doesn't support overloaded params for same param count + methodToParams.put("4.getCountsForFieldsInDateRange", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.getEarliestOccurrenceOfField", getScansPerCall(new Args[] { + new Args(new Object[] {"f1"}), + new Args(new Object[] {"f2"}), + new Args(new Object[] {"f3"})}, 1)); + methodToParams.put("2.getEarliestOccurrenceOfFieldWithType", getScansPerCall(new Args[] { + new Args(new Object[] {"f1", "type1"}), + new Args(new Object[] {"f2", "type1"}), + new Args(new Object[] {"f1", "type2"}), + new Args(new Object[] {"f2", "type2"})}, 1, (s)-> { + s.addScanIterator(anyObject()); + })); // @formatter:on // test that all methods are tested @@ -1427,19 +1493,24 @@ public void cachingConsistencyTest() throws Exception { // no need to verify private continue; } + if (Modifier.isProtected(method.getModifiers())) { + // no need to verify protected + continue; + } - // TODO delete this after all methods are added // isolate a single test - if (!method.getName().equals("loadAllFields")) { - continue; - } + // if (!method.getName().equals("getCountsByFieldInDayWithTypes")) { + // continue; + // } + // if (method.getParameterCount() != 2) { + // continue; + // } String lookupName = method.getParameterCount() + "." + method.getName(); // verify all non static methods are tested - assertNotNull("MetadataHelper method '" + method.getName() + "()' with " + method.getParameterCount() + " args not tested", methodToParams.get(lookupName)); - // TODO delete this after testing -// System.out.println(method.getName() + ": Testing"); + assertNotNull("MetadataHelper method '" + method.getName() + "()' with " + method.getParameterCount() + " args not tested", + methodToParams.get(lookupName)); // clear spring caches to prevent cross contamination between calls cleanupCache(); @@ -1461,9 +1532,9 @@ public void cachingConsistencyTest() throws Exception { // test cache value matches if it was supposed to be a cache hit if (expectation.cachedOf != null) { if (expectation.exactMatch) { - assertTrue("didn't get exact expected cached value", expectation.cachedOf.getResult() == result); + assertTrue(method.getName() + "didn't get exact expected cached value", expectation.cachedOf.getResult() == result); } else { - assertEquals("didn't get equivalent value", expectation.cachedOf.getResult(), result); + assertEquals(method.getName() + "didn't get equivalent value", expectation.cachedOf.getResult(), result); } } } @@ -1471,15 +1542,15 @@ public void cachingConsistencyTest() throws Exception { EasyMock.verify(getAccumuloClient()); verifyAll(); - System.out.println(method.getName() + ": OK"); - // reset mocks to use again EasyMock.reset(getAccumuloClient()); resetAll(); } } - // force cache clear + /** + * Reset the spring cache's to prevent contamination + */ private void cleanupCache() { for (String cacheName : cacheManager.getCacheNames()) { cacheManager.getCache(cacheName).clear(); @@ -1506,22 +1577,36 @@ private class ScannerExpectation implements Expectation { private final String table; private final int count; + private ExtraExpectation extra; private ScannerExpectation(String table, int count) { this.table = table; this.count = count; } + private ScannerExpectation(String table, int count, ExtraExpectation extra) { + this.table = table; + this.count = count; + this.extra = extra; + } + @Override public void expect() throws Exception { for (int i = 0; i < count; i++) { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner(table, s, entries); + + if (extra != null) { + extra.extraExpectation(s); + } } } } + /** + * Used to set expectations and cache settings of a method call. Used by cachingConsistencyTest() + */ private static class MethodParamsAndExpectations implements Expectation { public MetadataHelper metadataHelper; public Object[] args; @@ -1573,13 +1658,17 @@ public Object getResult() { /** * expect/mock the scanner creation and return of results from an iterator * - * @param table + * @param table if not null expects a specific table, otherwise any table * @param mockScanner * @param entries * @throws TableNotFoundException */ private void expectScanner(String table, Scanner mockScanner, List> entries) throws TableNotFoundException { - expect(getAccumuloClient().createScanner(eq(table), EasyMock.anyObject())).andReturn(mockScanner); + if (table != null) { + expect(getAccumuloClient().createScanner(eq(table), anyObject())).andReturn(mockScanner); + } else { + expect(getAccumuloClient().createScanner(isA(String.class), anyObject())).andReturn(mockScanner); + } mockScanner.setRange(anyObject()); mockScanner.fetchColumnFamily(isA(Text.class)); expectLastCall().anyTimes(); @@ -1600,6 +1689,9 @@ public String getData(String key) { @Autowired private TestCache c; + /** + * If this test fails the spring cache wiring is broken + */ @Test public void testCache() { assertEquals(c.getData("a"), c.getData("a"));