From 62cf5e0f9041f934147029cb33854502d5e6022e Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Fri, 29 May 2026 16:56:32 +0000 Subject: [PATCH 1/2] Added MultiAuthVisibilityFilter to user iterator package Accumulo Access provides an AccessEvaluator that accepts a Collection of Sets of Strings (effectively Collection). The existing user VisibilityFilter only accepts a single Authorizations object. This new visibility filter accepts a set of Authorizations in a single filter. --- .../core/clientImpl/access/BytesAccess.java | 15 + .../user/MultiAuthVisibilityFilter.java | 176 ++++++++++++ .../user/MultiAuthVisibilityFilterTest.java | 260 ++++++++++++++++++ .../test/functional/VisibilityIT.java | 45 +++ 4 files changed, 496 insertions(+) create mode 100644 core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java create mode 100644 core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java index 4c65621b0fc..21ce1698e05 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java @@ -20,6 +20,8 @@ import static java.nio.charset.StandardCharsets.ISO_8859_1; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -93,6 +95,19 @@ public boolean canAccess(byte[] expression) { } } + public static BytesEvaluator newEvaluator(Collection authsSet) { + Collection> convertedAuths = new ArrayList<>(); + for (Authorizations auths : authsSet) { + List bytesAuths = auths.getAuthorizations(); + Set stringAuths = new HashSet<>(bytesAuths.size()); + for (var auth : bytesAuths) { + stringAuths.add(new String(auth, ISO_8859_1)); + } + convertedAuths.add(stringAuths); + } + return new BytesEvaluator(ACCESS.newEvaluator(convertedAuths)); + } + public static BytesEvaluator newEvaluator(Authorizations auths) { List bytesAuths = auths.getAuthorizations(); Set stringAuths = new HashSet<>(bytesAuths.size()); diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java new file mode 100644 index 00000000000..b31e5cb07ab --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.iterators.user; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.Objects; + +import org.apache.accumulo.access.InvalidAccessExpressionException; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.clientImpl.access.BytesAccess; +import org.apache.accumulo.core.data.ArrayByteSequence; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.Filter; +import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.OptionDescriber; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.collections4.map.LRUMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +public class MultiAuthVisibilityFilter extends Filter implements OptionDescriber { + private BytesAccess.BytesEvaluator accessEvaluator; + protected Map cache; + private final ArrayByteSequence testVis = new ArrayByteSequence(new byte[0]); + + private static final Logger log = LoggerFactory.getLogger(VisibilityFilter.class); + + private static final String NUM_AUTHS = "numAuths"; + private static final String AUTH_PREFIX = "auth_"; + private static final String FILTER_INVALID_ONLY = "filterInvalid"; + + private boolean filterInvalid; + + @Override + public void init(SortedKeyValueIterator source, Map options, + IteratorEnvironment env) throws IOException { + super.init(source, options, env); + validateOptions(options); + this.filterInvalid = Boolean.parseBoolean(options.get(FILTER_INVALID_ONLY)); + + if (!filterInvalid) { + String numAuthsParameter = options.get(NUM_AUTHS); + Objects.requireNonNull(numAuthsParameter, "NUM_AUTHS option not set."); + int numAuths = Integer.parseInt(numAuthsParameter); + Preconditions.checkArgument(numAuths >= 0, NUM_AUTHS + " must be a positive integer"); + + Collection authSet = new ArrayList<>(); + if (numAuths == 0) { + authSet.add(new Authorizations()); + } else { + for (int idx = 0; idx < numAuths; idx++) { + String auths = options.get(AUTH_PREFIX + idx); + Authorizations authObj = auths == null || auths.isEmpty() ? new Authorizations() + : new Authorizations(auths.getBytes(UTF_8)); + authSet.add(authObj); + } + String auths = options.get(AUTH_PREFIX + numAuths); + Preconditions.checkArgument(auths == null, + "NUM_AUTHS is set incorrectly, should be at least: " + NUM_AUTHS + 1); + } + this.accessEvaluator = BytesAccess.newEvaluator(authSet); + } + this.cache = new LRUMap<>(1000); + } + + @Override + public SortedKeyValueIterator deepCopy(IteratorEnvironment env) { + MultiAuthVisibilityFilter result = (MultiAuthVisibilityFilter) super.deepCopy(env); + result.filterInvalid = this.filterInvalid; + result.accessEvaluator = this.accessEvaluator; + result.cache = this.cache; + return result; + } + + @Override + public boolean accept(Key k, Value v) { + // The following call will replace the contents of testVis + // with the bytes for the column visibility for k. Any cached + // version of testVis needs to be a copy to avoid modifying + // the cached version. + k.getColumnVisibilityData(testVis); + if (filterInvalid) { + Boolean b = cache.get(testVis); + if (b != null) { + return b; + } + final ArrayByteSequence copy = new ArrayByteSequence(testVis); + try { + BytesAccess.validate(copy.toArray()); + // cache a copy of testVis + cache.put(copy, true); + return true; + } catch (InvalidAccessExpressionException e) { + // cache a copy of testVis + cache.put(copy, false); + return false; + } + } else { + if (testVis.length() == 0) { + return true; + } + + Boolean b = cache.get(testVis); + if (b != null) { + return b; + } + + final ArrayByteSequence copy = new ArrayByteSequence(testVis); + try { + boolean bb = accessEvaluator.canAccess(copy.toArray()); + // cache a copy of testVis + cache.put(copy, bb); + return bb; + } catch (InvalidAccessExpressionException e) { + log.error("Parse Error with visibility of Key: {}", k, e); + return false; + } + } + } + + @Override + public IteratorOptions describeOptions() { + IteratorOptions io = super.describeOptions(); + io.setName("multiAuthVisibilityFilter"); + io.setDescription("The MultiAuthVisibilityFilter allows you to filter for key/value" + + " pairs by a collection of sets of authorizations or filter invalid labels from corrupt files."); + io.addNamedOption(FILTER_INVALID_ONLY, + "if 'true', the iterator is instructed to ignore the authorizations and" + + " only filter invalid visibility labels (default: false)"); + io.addNamedOption(NUM_AUTHS, + "The number of serialized authorizations to filter against (default 0)"); + io.addUnnamedOption(AUTH_PREFIX + + "N, where the value is a serialized set of authorizations. N must be between zero and NUM_AUTHS."); + return io; + } + + public static void setAuthorizations(IteratorSetting setting, Collection auths) { + setting.addOption(NUM_AUTHS, Integer.toString(auths.size())); + int idx = 0; + for (Authorizations auth : auths) { + setting.addOption(AUTH_PREFIX + idx, auth.serialize()); + idx++; + } + } + + public static void filterInvalidLabelsOnly(IteratorSetting setting, boolean featureEnabled) { + setting.addOption(FILTER_INVALID_ONLY, Boolean.toString(featureEnabled)); + } + +} diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java new file mode 100644 index 00000000000..8b4b1a217b6 --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.iterators.user; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.Filter; +import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.Test; + +public class MultiAuthVisibilityFilterTest { + + private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); + + private static final Text BAD = new Text("bad"); + private static final Text GOOD = new Text("good"); + private static final Text EMPTY_VIS = new Text(""); + private static final Text GOOD_VIS = new Text("abc|def"); + private static final Text HIDDEN_VIS = new Text("abc&def&ghi"); + private static final Text BAD_VIS = new Text("&"); + private static final Value EMPTY_VALUE = new Value(); + + private TreeMap createUnprotectedSource(int numPublic, int numHidden) { + TreeMap source = new TreeMap<>(); + for (int i = 0; i < numPublic; i++) { + source.put(new Key(new Text(String.format("%03d", i)), GOOD, GOOD, EMPTY_VIS), EMPTY_VALUE); + } + for (int i = 0; i < numHidden; i++) { + source.put(new Key(new Text(String.format("%03d", i)), BAD, BAD, GOOD_VIS), EMPTY_VALUE); + } + return source; + } + + private TreeMap createPollutedSource(int numGood, int numBad) { + TreeMap source = new TreeMap<>(); + for (int i = 0; i < numGood; i++) { + source.put(new Key(new Text(String.format("%03d", i)), GOOD, GOOD, GOOD_VIS), EMPTY_VALUE); + } + for (int i = 0; i < numBad; i++) { + source.put(new Key(new Text(String.format("%03d", i)), BAD, BAD, BAD_VIS), EMPTY_VALUE); + } + return source; + } + + private TreeMap createSourceWithHiddenData(int numViewable, int numHidden) { + TreeMap source = new TreeMap<>(); + for (int i = 0; i < numViewable; i++) { + source.put(new Key(new Text(String.format("%03d", i)), GOOD, GOOD, GOOD_VIS), EMPTY_VALUE); + } + for (int i = 0; i < numHidden; i++) { + source.put(new Key(new Text(String.format("%03d", i)), BAD, BAD, HIDDEN_VIS), EMPTY_VALUE); + } + return source; + } + + private void verify(TreeMap source, int expectedSourceSize, Map options, + Text expectedCF, Text expectedCQ, Text expectedCV, int expectedFinalCount) + throws IOException { + assertEquals(expectedSourceSize, source.size()); + + Filter filter = new MultiAuthVisibilityFilter(); + filter.init(new SortedMapIterator(source), options, null); + filter.seek(new Range(), EMPTY_COL_FAMS, false); + + int count = 0; + while (filter.hasTop()) { + count++; + // System.out.println(DefaultFormatter.formatEntry( + // Collections.singletonMap(filter.getTopKey(), + // filter.getTopValue()).entrySet().iterator().next(), + // false)); + assertEquals(expectedCF, filter.getTopKey().getColumnFamily()); + assertEquals(expectedCQ, filter.getTopKey().getColumnQualifier()); + assertEquals(expectedCV, filter.getTopKey().getColumnVisibility()); + filter.next(); + } + assertEquals(expectedFinalCount, count); + } + + @Test + public void testAllowValidLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.filterInvalidLabelsOnly(is, true); + + TreeMap source = createPollutedSource(1, 2); + verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); + + source = createPollutedSource(30, 500); + verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30); + + source = createPollutedSource(1000, 500); + verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000); + } + + @Test + public void testAllowBadLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setNegate(is, true); + MultiAuthVisibilityFilter.filterInvalidLabelsOnly(is, true); + + TreeMap source = createPollutedSource(1, 2); + verify(source, 3, is.getOptions(), BAD, BAD, BAD_VIS, 2); + + source = createPollutedSource(30, 500); + verify(source, 530, is.getOptions(), BAD, BAD, BAD_VIS, 500); + + source = createPollutedSource(1000, 500); + verify(source, 1500, is.getOptions(), BAD, BAD, BAD_VIS, 500); + } + + @Test + public void testAllowAuthorizedLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("def"))); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000); + } + + @Test + public void testAllowUnauthorizedLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setNegate(is, true); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("def"))); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, is.getOptions(), BAD, BAD, HIDDEN_VIS, 2); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); + } + + @Test + public void testNoLabels() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setNegate(is, false); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations())); + + TreeMap source = createUnprotectedSource(5, 2); + verify(source, 7, is.getOptions(), GOOD, GOOD, EMPTY_VIS, 5); + + MultiAuthVisibilityFilter.setNegate(is, true); + verify(source, 7, is.getOptions(), BAD, BAD, GOOD_VIS, 2); + } + + @Test + public void testFilterUnauthorizedAndBad() throws IOException { + /* + * if not explicitly filtering bad labels, they will still be filtered while validating against + * authorizations, but it will be very verbose in the logs + */ + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("def"))); + + TreeMap source = createSourceWithHiddenData(1, 5); + for (Entry entry : createPollutedSource(0, 1).entrySet()) { + source.put(entry.getKey(), entry.getValue()); + } + + verify(source, 7, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); + } + + @Test + public void testCommaSeparatedAuthorizations() throws IOException { + Map options = Map.of("numAuths", "1", "auth_0", "x,def,y"); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, options, GOOD, GOOD, GOOD_VIS, 1); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, options, GOOD, GOOD, GOOD_VIS, 30); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, options, GOOD, GOOD, GOOD_VIS, 1000); + } + + @Test + public void testSerializedAuthorizations() throws IOException { + Map options = + Map.of("numAuths", "1", "auth_0", new Authorizations("x", "def", "y").serialize()); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, options, GOOD, GOOD, GOOD_VIS, 1); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, options, GOOD, GOOD, GOOD_VIS, 30); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, options, GOOD, GOOD, GOOD_VIS, 1000); + } + + @Test + public void testStaticConfigurators() { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.filterInvalidLabelsOnly(is, false); + MultiAuthVisibilityFilter.setNegate(is, true); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("abc", "def"))); + + Map opts = is.getOptions(); + assertEquals("false", opts.get("filterInvalid")); + assertEquals("true", opts.get("negate")); + assertEquals("1", opts.get("numAuths")); + assertEquals(new Authorizations("abc", "def").serialize(), opts.get("auth_0")); + } + + @Test + public void testDeepCopyAfterInit() throws IOException { + IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("abc"))); + Map opts = is.getOptions(); + Filter filter = new MultiAuthVisibilityFilter(); + TreeMap source = new TreeMap<>(); + filter.init(new SortedMapIterator(source), opts, null); + Filter copyFilter = (Filter) filter.deepCopy(null); + Key k = new Key("row", "cf", "cq", "abc"); + assertTrue(copyFilter.accept(k, new Value())); + } + +} diff --git a/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java b/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java index 96cc472f8c7..32ea3ea5c8c 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java @@ -37,12 +37,14 @@ import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.MultiAuthVisibilityFilter; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.util.ByteArraySet; @@ -92,6 +94,7 @@ public void run() throws Exception { insertData(c, table); queryData(c, table); + queryDataMultiAuth(c, table); deleteData(c, table); insertDefaultData(c, table2); @@ -222,6 +225,48 @@ private void queryData(AccumuloClient c, String tableName) throws Exception { queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss(), expected); } + /** + * Configures Scanners with the users default authorizations, then it adds a + * MultiAuthVisibilityFilter with different sets of Authorizations + */ + private void queryDataMultiAuth(AccumuloClient c, String tableName) throws Exception { + + c.securityOperations().changeUserAuthorizations(getAdminPrincipal(), + new Authorizations("A", "B", "FOO", "L", "M", "Z")); + + Authorizations userAuths = c.securityOperations().getUserAuthorizations(c.whoami()); + + Set expectedUserAuths = + Set.of("v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13"); + try (Scanner scanner = c.createScanner(tableName, userAuths); + BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) { + verify(scanner.iterator(), expectedUserAuths.toArray(new String[] {})); + + bs.setRanges(Collections.singleton(new Range())); + verify(bs.iterator(), expectedUserAuths.toArray(new String[] {})); + } + + Authorizations entity1 = new Authorizations("A", "B", "FOO", "L", "M"); + Authorizations entity2 = new Authorizations("B", "FOO", "Z"); + // should only see entries with no column visibility, B and/or FOO + Set expectedAuths = Set.of("v1", "v3", "v11"); + + IteratorSetting is = new IteratorSetting(100, "userAuths", MultiAuthVisibilityFilter.class); + MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(entity1, entity2)); + + try (Scanner scanner = c.createScanner(tableName, userAuths); + BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) { + + scanner.addScanIterator(is); + verify(scanner.iterator(), expectedAuths.toArray(new String[] {})); + + bs.setRanges(Collections.singleton(new Range())); + bs.addScanIterator(is); + verify(bs.iterator(), expectedAuths.toArray(new String[] {})); + } + + } + private void queryData(AccumuloClient c, String tableName, Set allAuths, Set userAuths, Map,Set> expected) throws Exception { From 8cc66494129a45471c2d449c20e9670b813a8515 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Mon, 8 Jun 2026 14:39:22 +0000 Subject: [PATCH 2/2] Merged multi-auth feature into user VisibilityFilter --- .../user/MultiAuthVisibilityFilter.java | 176 ------------ .../core/iterators/user/VisibilityFilter.java | 52 +++- .../user/MultiAuthVisibilityFilterTest.java | 260 ------------------ .../iterators/user/VisibilityFilterTest.java | 41 ++- .../test/functional/VisibilityIT.java | 6 +- 5 files changed, 82 insertions(+), 453 deletions(-) delete mode 100644 core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java delete mode 100644 core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java deleted file mode 100644 index b31e5cb07ab..00000000000 --- a/core/src/main/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilter.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.core.iterators.user; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Map; -import java.util.Objects; - -import org.apache.accumulo.access.InvalidAccessExpressionException; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.clientImpl.access.BytesAccess; -import org.apache.accumulo.core.data.ArrayByteSequence; -import org.apache.accumulo.core.data.ByteSequence; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.Filter; -import org.apache.accumulo.core.iterators.IteratorEnvironment; -import org.apache.accumulo.core.iterators.OptionDescriber; -import org.apache.accumulo.core.iterators.SortedKeyValueIterator; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.commons.collections4.map.LRUMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Preconditions; - -public class MultiAuthVisibilityFilter extends Filter implements OptionDescriber { - private BytesAccess.BytesEvaluator accessEvaluator; - protected Map cache; - private final ArrayByteSequence testVis = new ArrayByteSequence(new byte[0]); - - private static final Logger log = LoggerFactory.getLogger(VisibilityFilter.class); - - private static final String NUM_AUTHS = "numAuths"; - private static final String AUTH_PREFIX = "auth_"; - private static final String FILTER_INVALID_ONLY = "filterInvalid"; - - private boolean filterInvalid; - - @Override - public void init(SortedKeyValueIterator source, Map options, - IteratorEnvironment env) throws IOException { - super.init(source, options, env); - validateOptions(options); - this.filterInvalid = Boolean.parseBoolean(options.get(FILTER_INVALID_ONLY)); - - if (!filterInvalid) { - String numAuthsParameter = options.get(NUM_AUTHS); - Objects.requireNonNull(numAuthsParameter, "NUM_AUTHS option not set."); - int numAuths = Integer.parseInt(numAuthsParameter); - Preconditions.checkArgument(numAuths >= 0, NUM_AUTHS + " must be a positive integer"); - - Collection authSet = new ArrayList<>(); - if (numAuths == 0) { - authSet.add(new Authorizations()); - } else { - for (int idx = 0; idx < numAuths; idx++) { - String auths = options.get(AUTH_PREFIX + idx); - Authorizations authObj = auths == null || auths.isEmpty() ? new Authorizations() - : new Authorizations(auths.getBytes(UTF_8)); - authSet.add(authObj); - } - String auths = options.get(AUTH_PREFIX + numAuths); - Preconditions.checkArgument(auths == null, - "NUM_AUTHS is set incorrectly, should be at least: " + NUM_AUTHS + 1); - } - this.accessEvaluator = BytesAccess.newEvaluator(authSet); - } - this.cache = new LRUMap<>(1000); - } - - @Override - public SortedKeyValueIterator deepCopy(IteratorEnvironment env) { - MultiAuthVisibilityFilter result = (MultiAuthVisibilityFilter) super.deepCopy(env); - result.filterInvalid = this.filterInvalid; - result.accessEvaluator = this.accessEvaluator; - result.cache = this.cache; - return result; - } - - @Override - public boolean accept(Key k, Value v) { - // The following call will replace the contents of testVis - // with the bytes for the column visibility for k. Any cached - // version of testVis needs to be a copy to avoid modifying - // the cached version. - k.getColumnVisibilityData(testVis); - if (filterInvalid) { - Boolean b = cache.get(testVis); - if (b != null) { - return b; - } - final ArrayByteSequence copy = new ArrayByteSequence(testVis); - try { - BytesAccess.validate(copy.toArray()); - // cache a copy of testVis - cache.put(copy, true); - return true; - } catch (InvalidAccessExpressionException e) { - // cache a copy of testVis - cache.put(copy, false); - return false; - } - } else { - if (testVis.length() == 0) { - return true; - } - - Boolean b = cache.get(testVis); - if (b != null) { - return b; - } - - final ArrayByteSequence copy = new ArrayByteSequence(testVis); - try { - boolean bb = accessEvaluator.canAccess(copy.toArray()); - // cache a copy of testVis - cache.put(copy, bb); - return bb; - } catch (InvalidAccessExpressionException e) { - log.error("Parse Error with visibility of Key: {}", k, e); - return false; - } - } - } - - @Override - public IteratorOptions describeOptions() { - IteratorOptions io = super.describeOptions(); - io.setName("multiAuthVisibilityFilter"); - io.setDescription("The MultiAuthVisibilityFilter allows you to filter for key/value" - + " pairs by a collection of sets of authorizations or filter invalid labels from corrupt files."); - io.addNamedOption(FILTER_INVALID_ONLY, - "if 'true', the iterator is instructed to ignore the authorizations and" - + " only filter invalid visibility labels (default: false)"); - io.addNamedOption(NUM_AUTHS, - "The number of serialized authorizations to filter against (default 0)"); - io.addUnnamedOption(AUTH_PREFIX - + "N, where the value is a serialized set of authorizations. N must be between zero and NUM_AUTHS."); - return io; - } - - public static void setAuthorizations(IteratorSetting setting, Collection auths) { - setting.addOption(NUM_AUTHS, Integer.toString(auths.size())); - int idx = 0; - for (Authorizations auth : auths) { - setting.addOption(AUTH_PREFIX + idx, auth.serialize()); - idx++; - } - } - - public static void filterInvalidLabelsOnly(IteratorSetting setting, boolean featureEnabled) { - setting.addOption(FILTER_INVALID_ONLY, Boolean.toString(featureEnabled)); - } - -} diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java index 92e5079ab65..a530920c4b2 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java @@ -21,7 +21,10 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.Map; +import java.util.Objects; import org.apache.accumulo.access.InvalidAccessExpressionException; import org.apache.accumulo.core.client.IteratorSetting; @@ -39,6 +42,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Preconditions; + /** * A SortedKeyValueIterator that filters based on ColumnVisibility. */ @@ -50,7 +55,8 @@ public class VisibilityFilter extends Filter implements OptionDescriber { private static final Logger log = LoggerFactory.getLogger(VisibilityFilter.class); - private static final String AUTHS = "auths"; + private static final String NUM_AUTHS = "numAuths"; + private static final String AUTH_PREFIX = "auth_"; private static final String FILTER_INVALID_ONLY = "filterInvalid"; private boolean filterInvalid; @@ -63,11 +69,26 @@ public void init(SortedKeyValueIterator source, Map op this.filterInvalid = Boolean.parseBoolean(options.get(FILTER_INVALID_ONLY)); if (!filterInvalid) { - String auths = options.get(AUTHS); - Authorizations authObj = auths == null || auths.isEmpty() ? new Authorizations() - : new Authorizations(auths.getBytes(UTF_8)); - - this.accessEvaluator = BytesAccess.newEvaluator(authObj); + String numAuthsParameter = options.get(NUM_AUTHS); + Objects.requireNonNull(numAuthsParameter, "NUM_AUTHS option not set."); + int numAuths = Integer.parseInt(numAuthsParameter); + Preconditions.checkArgument(numAuths >= 0, NUM_AUTHS + " must be a positive integer"); + + Collection authSet = new ArrayList<>(); + if (numAuths == 0) { + authSet.add(new Authorizations()); + } else { + for (int idx = 0; idx < numAuths; idx++) { + String auths = options.get(AUTH_PREFIX + idx); + Authorizations authObj = auths == null || auths.isEmpty() ? new Authorizations() + : new Authorizations(auths.getBytes(UTF_8)); + authSet.add(authObj); + } + String auths = options.get(AUTH_PREFIX + numAuths); + Preconditions.checkArgument(auths == null, + "NUM_AUTHS is set incorrectly, should be at least: " + NUM_AUTHS + " = " + 1); + } + this.accessEvaluator = BytesAccess.newEvaluator(authSet); } this.cache = new LRUMap<>(1000); } @@ -136,14 +157,25 @@ public IteratorOptions describeOptions() { io.addNamedOption(FILTER_INVALID_ONLY, "if 'true', the iterator is instructed to ignore the authorizations and" + " only filter invalid visibility labels (default: false)"); - io.addNamedOption(AUTHS, - "the serialized set of authorizations to filter against (default: empty" - + " string, accepts only entries visible by all)"); + io.addNamedOption(NUM_AUTHS, + "The number of serialized authorizations to filter against (default 0)"); + io.addUnnamedOption(AUTH_PREFIX + + "N, where the value is a serialized set of authorizations. N must be between zero and NUM_AUTHS."); return io; } public static void setAuthorizations(IteratorSetting setting, Authorizations auths) { - setting.addOption(AUTHS, auths.serialize()); + setting.addOption(NUM_AUTHS, "1"); + setting.addOption(AUTH_PREFIX + 0, auths.serialize()); + } + + public static void setAuthorizations(IteratorSetting setting, Collection auths) { + setting.addOption(NUM_AUTHS, Integer.toString(auths.size())); + int idx = 0; + for (Authorizations auth : auths) { + setting.addOption(AUTH_PREFIX + idx, auth.serialize()); + idx++; + } } public static void filterInvalidLabelsOnly(IteratorSetting setting, boolean featureEnabled) { diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java deleted file mode 100644 index 8b4b1a217b6..00000000000 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/MultiAuthVisibilityFilterTest.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.core.iterators.user; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeMap; - -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.data.ByteSequence; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.Filter; -import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.hadoop.io.Text; -import org.junit.jupiter.api.Test; - -public class MultiAuthVisibilityFilterTest { - - private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); - - private static final Text BAD = new Text("bad"); - private static final Text GOOD = new Text("good"); - private static final Text EMPTY_VIS = new Text(""); - private static final Text GOOD_VIS = new Text("abc|def"); - private static final Text HIDDEN_VIS = new Text("abc&def&ghi"); - private static final Text BAD_VIS = new Text("&"); - private static final Value EMPTY_VALUE = new Value(); - - private TreeMap createUnprotectedSource(int numPublic, int numHidden) { - TreeMap source = new TreeMap<>(); - for (int i = 0; i < numPublic; i++) { - source.put(new Key(new Text(String.format("%03d", i)), GOOD, GOOD, EMPTY_VIS), EMPTY_VALUE); - } - for (int i = 0; i < numHidden; i++) { - source.put(new Key(new Text(String.format("%03d", i)), BAD, BAD, GOOD_VIS), EMPTY_VALUE); - } - return source; - } - - private TreeMap createPollutedSource(int numGood, int numBad) { - TreeMap source = new TreeMap<>(); - for (int i = 0; i < numGood; i++) { - source.put(new Key(new Text(String.format("%03d", i)), GOOD, GOOD, GOOD_VIS), EMPTY_VALUE); - } - for (int i = 0; i < numBad; i++) { - source.put(new Key(new Text(String.format("%03d", i)), BAD, BAD, BAD_VIS), EMPTY_VALUE); - } - return source; - } - - private TreeMap createSourceWithHiddenData(int numViewable, int numHidden) { - TreeMap source = new TreeMap<>(); - for (int i = 0; i < numViewable; i++) { - source.put(new Key(new Text(String.format("%03d", i)), GOOD, GOOD, GOOD_VIS), EMPTY_VALUE); - } - for (int i = 0; i < numHidden; i++) { - source.put(new Key(new Text(String.format("%03d", i)), BAD, BAD, HIDDEN_VIS), EMPTY_VALUE); - } - return source; - } - - private void verify(TreeMap source, int expectedSourceSize, Map options, - Text expectedCF, Text expectedCQ, Text expectedCV, int expectedFinalCount) - throws IOException { - assertEquals(expectedSourceSize, source.size()); - - Filter filter = new MultiAuthVisibilityFilter(); - filter.init(new SortedMapIterator(source), options, null); - filter.seek(new Range(), EMPTY_COL_FAMS, false); - - int count = 0; - while (filter.hasTop()) { - count++; - // System.out.println(DefaultFormatter.formatEntry( - // Collections.singletonMap(filter.getTopKey(), - // filter.getTopValue()).entrySet().iterator().next(), - // false)); - assertEquals(expectedCF, filter.getTopKey().getColumnFamily()); - assertEquals(expectedCQ, filter.getTopKey().getColumnQualifier()); - assertEquals(expectedCV, filter.getTopKey().getColumnVisibility()); - filter.next(); - } - assertEquals(expectedFinalCount, count); - } - - @Test - public void testAllowValidLabelsOnly() throws IOException { - IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); - MultiAuthVisibilityFilter.filterInvalidLabelsOnly(is, true); - - TreeMap source = createPollutedSource(1, 2); - verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); - - source = createPollutedSource(30, 500); - verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30); - - source = createPollutedSource(1000, 500); - verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000); - } - - @Test - public void testAllowBadLabelsOnly() throws IOException { - IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); - MultiAuthVisibilityFilter.setNegate(is, true); - MultiAuthVisibilityFilter.filterInvalidLabelsOnly(is, true); - - TreeMap source = createPollutedSource(1, 2); - verify(source, 3, is.getOptions(), BAD, BAD, BAD_VIS, 2); - - source = createPollutedSource(30, 500); - verify(source, 530, is.getOptions(), BAD, BAD, BAD_VIS, 500); - - source = createPollutedSource(1000, 500); - verify(source, 1500, is.getOptions(), BAD, BAD, BAD_VIS, 500); - } - - @Test - public void testAllowAuthorizedLabelsOnly() throws IOException { - IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); - MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("def"))); - - TreeMap source = createSourceWithHiddenData(1, 2); - verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); - - source = createSourceWithHiddenData(30, 500); - verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30); - - source = createSourceWithHiddenData(1000, 500); - verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000); - } - - @Test - public void testAllowUnauthorizedLabelsOnly() throws IOException { - IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); - MultiAuthVisibilityFilter.setNegate(is, true); - MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("def"))); - - TreeMap source = createSourceWithHiddenData(1, 2); - verify(source, 3, is.getOptions(), BAD, BAD, HIDDEN_VIS, 2); - - source = createSourceWithHiddenData(30, 500); - verify(source, 530, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); - - source = createSourceWithHiddenData(1000, 500); - verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); - } - - @Test - public void testNoLabels() throws IOException { - IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); - MultiAuthVisibilityFilter.setNegate(is, false); - MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations())); - - TreeMap source = createUnprotectedSource(5, 2); - verify(source, 7, is.getOptions(), GOOD, GOOD, EMPTY_VIS, 5); - - MultiAuthVisibilityFilter.setNegate(is, true); - verify(source, 7, is.getOptions(), BAD, BAD, GOOD_VIS, 2); - } - - @Test - public void testFilterUnauthorizedAndBad() throws IOException { - /* - * if not explicitly filtering bad labels, they will still be filtered while validating against - * authorizations, but it will be very verbose in the logs - */ - IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); - MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("def"))); - - TreeMap source = createSourceWithHiddenData(1, 5); - for (Entry entry : createPollutedSource(0, 1).entrySet()) { - source.put(entry.getKey(), entry.getValue()); - } - - verify(source, 7, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); - } - - @Test - public void testCommaSeparatedAuthorizations() throws IOException { - Map options = Map.of("numAuths", "1", "auth_0", "x,def,y"); - - TreeMap source = createSourceWithHiddenData(1, 2); - verify(source, 3, options, GOOD, GOOD, GOOD_VIS, 1); - - source = createSourceWithHiddenData(30, 500); - verify(source, 530, options, GOOD, GOOD, GOOD_VIS, 30); - - source = createSourceWithHiddenData(1000, 500); - verify(source, 1500, options, GOOD, GOOD, GOOD_VIS, 1000); - } - - @Test - public void testSerializedAuthorizations() throws IOException { - Map options = - Map.of("numAuths", "1", "auth_0", new Authorizations("x", "def", "y").serialize()); - - TreeMap source = createSourceWithHiddenData(1, 2); - verify(source, 3, options, GOOD, GOOD, GOOD_VIS, 1); - - source = createSourceWithHiddenData(30, 500); - verify(source, 530, options, GOOD, GOOD, GOOD_VIS, 30); - - source = createSourceWithHiddenData(1000, 500); - verify(source, 1500, options, GOOD, GOOD, GOOD_VIS, 1000); - } - - @Test - public void testStaticConfigurators() { - IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); - MultiAuthVisibilityFilter.filterInvalidLabelsOnly(is, false); - MultiAuthVisibilityFilter.setNegate(is, true); - MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("abc", "def"))); - - Map opts = is.getOptions(); - assertEquals("false", opts.get("filterInvalid")); - assertEquals("true", opts.get("negate")); - assertEquals("1", opts.get("numAuths")); - assertEquals(new Authorizations("abc", "def").serialize(), opts.get("auth_0")); - } - - @Test - public void testDeepCopyAfterInit() throws IOException { - IteratorSetting is = new IteratorSetting(1, MultiAuthVisibilityFilter.class); - MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(new Authorizations("abc"))); - Map opts = is.getOptions(); - Filter filter = new MultiAuthVisibilityFilter(); - TreeMap source = new TreeMap<>(); - filter.init(new SortedMapIterator(source), opts, null); - Filter copyFilter = (Filter) filter.deepCopy(null); - Key k = new Key("row", "cf", "cq", "abc"); - assertTrue(copyFilter.accept(k, new Value())); - } - -} diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java index 075e2bef330..f9db84de84b 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java @@ -24,7 +24,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; @@ -155,6 +155,22 @@ public void testAllowAuthorizedLabelsOnly() throws IOException { verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000); } + @Test + public void testMulitAllowAuthorizedLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class); + VisibilityFilter.setAuthorizations(is, + List.of(new Authorizations("abc"), new Authorizations("def"))); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000); + } + @Test public void testAllowUnauthorizedLabelsOnly() throws IOException { IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class); @@ -171,6 +187,23 @@ public void testAllowUnauthorizedLabelsOnly() throws IOException { verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); } + @Test + public void testMultiAllowUnauthorizedLabelsOnly() throws IOException { + IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class); + VisibilityFilter.setNegate(is, true); + VisibilityFilter.setAuthorizations(is, + List.of(new Authorizations("abc"), new Authorizations("def"))); + + TreeMap source = createSourceWithHiddenData(1, 2); + verify(source, 3, is.getOptions(), BAD, BAD, HIDDEN_VIS, 2); + + source = createSourceWithHiddenData(30, 500); + verify(source, 530, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); + + source = createSourceWithHiddenData(1000, 500); + verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500); + } + @Test public void testNoLabels() throws IOException { IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class); @@ -203,7 +236,7 @@ public void testFilterUnauthorizedAndBad() throws IOException { @Test public void testCommaSeparatedAuthorizations() throws IOException { - Map options = Collections.singletonMap("auths", "x,def,y"); + Map options = Map.of("numAuths", "1", "auth_0", "x,def,y"); TreeMap source = createSourceWithHiddenData(1, 2); verify(source, 3, options, GOOD, GOOD, GOOD_VIS, 1); @@ -218,7 +251,7 @@ public void testCommaSeparatedAuthorizations() throws IOException { @Test public void testSerializedAuthorizations() throws IOException { Map options = - Collections.singletonMap("auths", new Authorizations("x", "def", "y").serialize()); + Map.of("numAuths", "1", "auth_0", new Authorizations("x", "def", "y").serialize()); TreeMap source = createSourceWithHiddenData(1, 2); verify(source, 3, options, GOOD, GOOD, GOOD_VIS, 1); @@ -240,7 +273,7 @@ public void testStaticConfigurators() { Map opts = is.getOptions(); assertEquals("false", opts.get("filterInvalid")); assertEquals("true", opts.get("negate")); - assertEquals(new Authorizations("abc", "def").serialize(), opts.get("auths")); + assertEquals(new Authorizations("abc", "def").serialize(), opts.get("auth_0")); } @Test diff --git a/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java b/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java index 32ea3ea5c8c..c6bed4b1f79 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java @@ -44,7 +44,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.user.MultiAuthVisibilityFilter; +import org.apache.accumulo.core.iterators.user.VisibilityFilter; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.core.util.ByteArraySet; @@ -251,8 +251,8 @@ private void queryDataMultiAuth(AccumuloClient c, String tableName) throws Excep // should only see entries with no column visibility, B and/or FOO Set expectedAuths = Set.of("v1", "v3", "v11"); - IteratorSetting is = new IteratorSetting(100, "userAuths", MultiAuthVisibilityFilter.class); - MultiAuthVisibilityFilter.setAuthorizations(is, Set.of(entity1, entity2)); + IteratorSetting is = new IteratorSetting(100, "userAuths", VisibilityFilter.class); + VisibilityFilter.setAuthorizations(is, Set.of(entity1, entity2)); try (Scanner scanner = c.createScanner(tableName, userAuths); BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) {