Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.thrift2;
019
020import static java.nio.ByteBuffer.wrap;
021import static org.junit.Assert.assertArrayEquals;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.nio.ByteBuffer;
028import java.security.PrivilegedExceptionAction;
029import java.util.ArrayList;
030import java.util.Collections;
031import java.util.Comparator;
032import java.util.List;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.hbase.HBaseClassTestRule;
035import org.apache.hadoop.hbase.HBaseTestingUtility;
036import org.apache.hadoop.hbase.HColumnDescriptor;
037import org.apache.hadoop.hbase.HTableDescriptor;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.Admin;
040import org.apache.hadoop.hbase.client.Connection;
041import org.apache.hadoop.hbase.client.ConnectionFactory;
042import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
043import org.apache.hadoop.hbase.security.User;
044import org.apache.hadoop.hbase.security.UserProvider;
045import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
046import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
047import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
048import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
049import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil;
050import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
051import org.apache.hadoop.hbase.testclassification.ClientTests;
052import org.apache.hadoop.hbase.testclassification.MediumTests;
053import org.apache.hadoop.hbase.thrift2.generated.TAppend;
054import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
055import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
056import org.apache.hadoop.hbase.thrift2.generated.TColumn;
057import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
058import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
059import org.apache.hadoop.hbase.thrift2.generated.TGet;
060import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
061import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
062import org.apache.hadoop.hbase.thrift2.generated.TPut;
063import org.apache.hadoop.hbase.thrift2.generated.TResult;
064import org.apache.hadoop.hbase.thrift2.generated.TScan;
065import org.apache.hadoop.hbase.util.Bytes;
066import org.junit.AfterClass;
067import org.junit.Assert;
068import org.junit.Before;
069import org.junit.BeforeClass;
070import org.junit.ClassRule;
071import org.junit.Test;
072import org.junit.experimental.categories.Category;
073import org.slf4j.Logger;
074import org.slf4j.LoggerFactory;
075
076@Category({ ClientTests.class, MediumTests.class })
077public class TestThriftHBaseServiceHandlerWithLabels {
078
079  @ClassRule
080  public static final HBaseClassTestRule CLASS_RULE =
081    HBaseClassTestRule.forClass(TestThriftHBaseServiceHandlerWithLabels.class);
082
083  private static final Logger LOG =
084    LoggerFactory.getLogger(TestThriftHBaseServiceHandlerWithLabels.class);
085  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
086
087  // Static names for tables, columns, rows, and values
088  private static byte[] tableAname = Bytes.toBytes("tableA");
089  private static byte[] familyAname = Bytes.toBytes("familyA");
090  private static byte[] familyBname = Bytes.toBytes("familyB");
091  private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
092  private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
093  private static byte[] valueAname = Bytes.toBytes("valueA");
094  private static byte[] valueBname = Bytes.toBytes("valueB");
095  private static HColumnDescriptor[] families =
096    new HColumnDescriptor[] { new HColumnDescriptor(familyAname).setMaxVersions(3),
097      new HColumnDescriptor(familyBname).setMaxVersions(2) };
098
099  private final static String TOPSECRET = "topsecret";
100  private final static String PUBLIC = "public";
101  private final static String PRIVATE = "private";
102  private final static String CONFIDENTIAL = "confidential";
103  private final static String SECRET = "secret";
104  private static User SUPERUSER;
105
106  private static Configuration conf;
107
108  public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
109    List<TColumnValue> columnValuesB) {
110    assertEquals(columnValuesA.size(), columnValuesB.size());
111    Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
112      @Override
113      public int compare(TColumnValue o1, TColumnValue o2) {
114        return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
115          Bytes.add(o2.getFamily(), o2.getQualifier()));
116      }
117    };
118    Collections.sort(columnValuesA, comparator);
119    Collections.sort(columnValuesB, comparator);
120
121    for (int i = 0; i < columnValuesA.size(); i++) {
122      TColumnValue a = columnValuesA.get(i);
123      TColumnValue b = columnValuesB.get(i);
124      assertArrayEquals(a.getFamily(), b.getFamily());
125      assertArrayEquals(a.getQualifier(), b.getQualifier());
126      assertArrayEquals(a.getValue(), b.getValue());
127    }
128  }
129
130  @BeforeClass
131  public static void beforeClass() throws Exception {
132
133    SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" });
134    conf = UTIL.getConfiguration();
135    conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
136      ScanLabelGenerator.class);
137    conf.set("hbase.superuser", SUPERUSER.getShortName());
138    VisibilityTestUtil.enableVisiblityLabels(conf);
139    UTIL.startMiniCluster(1);
140    // Wait for the labels table to become available
141    UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
142    createLabels();
143    Admin admin = UTIL.getAdmin();
144    HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableAname));
145    for (HColumnDescriptor family : families) {
146      tableDescriptor.addFamily(family);
147    }
148    admin.createTable(tableDescriptor);
149    admin.close();
150    setAuths();
151  }
152
153  private static void createLabels() throws IOException, InterruptedException {
154    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
155      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
156        @Override
157        public VisibilityLabelsResponse run() throws Exception {
158          String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
159          try (Connection conn = ConnectionFactory.createConnection(conf)) {
160            VisibilityClient.addLabels(conn, labels);
161          } catch (Throwable t) {
162            throw new IOException(t);
163          }
164          return null;
165        }
166      };
167    SUPERUSER.runAs(action);
168  }
169
170  private static void setAuths() throws IOException {
171    String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
172    try {
173      VisibilityClient.setAuths(UTIL.getConnection(), labels, User.getCurrent().getShortName());
174    } catch (Throwable t) {
175      throw new IOException(t);
176    }
177  }
178
179  @AfterClass
180  public static void afterClass() throws Exception {
181    UTIL.shutdownMiniCluster();
182  }
183
184  @Before
185  public void setup() throws Exception {
186
187  }
188
189  private ThriftHBaseServiceHandler createHandler() throws IOException {
190    return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
191  }
192
193  @Test
194  public void testScanWithVisibilityLabels() throws Exception {
195    ThriftHBaseServiceHandler handler = createHandler();
196    ByteBuffer table = wrap(tableAname);
197
198    // insert data
199    TColumnValue columnValue =
200      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
201    List<TColumnValue> columnValues = new ArrayList<>(1);
202    columnValues.add(columnValue);
203    for (int i = 0; i < 10; i++) {
204      TPut put = new TPut(wrap(Bytes.toBytes("testScan" + i)), columnValues);
205      if (i == 5) {
206        put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
207      } else {
208        put.setCellVisibility(new TCellVisibility()
209          .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
210      }
211      handler.put(table, put);
212    }
213
214    // create scan instance
215    TScan scan = new TScan();
216    List<TColumn> columns = new ArrayList<>(1);
217    TColumn column = new TColumn();
218    column.setFamily(familyAname);
219    column.setQualifier(qualifierAname);
220    columns.add(column);
221    scan.setColumns(columns);
222    scan.setStartRow(Bytes.toBytes("testScan"));
223    scan.setStopRow(Bytes.toBytes("testScan\uffff"));
224
225    TAuthorization tauth = new TAuthorization();
226    List<String> labels = new ArrayList<>(2);
227    labels.add(SECRET);
228    labels.add(PRIVATE);
229    tauth.setLabels(labels);
230    scan.setAuthorizations(tauth);
231    // get scanner and rows
232    int scanId = handler.openScanner(table, scan);
233    List<TResult> results = handler.getScannerRows(scanId, 10);
234    assertEquals(9, results.size());
235    Assert.assertFalse(Bytes.equals(results.get(5).getRow(), Bytes.toBytes("testScan" + 5)));
236    for (int i = 0; i < 9; i++) {
237      if (i < 5) {
238        assertArrayEquals(Bytes.toBytes("testScan" + i), results.get(i).getRow());
239      } else if (i == 5) {
240        continue;
241      } else {
242        assertArrayEquals(Bytes.toBytes("testScan" + (i + 1)), results.get(i).getRow());
243      }
244    }
245
246    // check that we are at the end of the scan
247    results = handler.getScannerRows(scanId, 9);
248    assertEquals(0, results.size());
249
250    // close scanner and check that it was indeed closed
251    handler.closeScanner(scanId);
252    try {
253      handler.getScannerRows(scanId, 9);
254      fail("Scanner id should be invalid");
255    } catch (TIllegalArgument e) {
256    }
257  }
258
259  @Test
260  public void testGetScannerResultsWithAuthorizations() throws Exception {
261    ThriftHBaseServiceHandler handler = createHandler();
262    ByteBuffer table = wrap(tableAname);
263
264    // insert data
265    TColumnValue columnValue =
266      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
267    List<TColumnValue> columnValues = new ArrayList<>(1);
268    columnValues.add(columnValue);
269    for (int i = 0; i < 20; i++) {
270      TPut put =
271        new TPut(wrap(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2))), columnValues);
272      if (i == 3) {
273        put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
274      } else {
275        put.setCellVisibility(new TCellVisibility()
276          .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
277      }
278      handler.put(table, put);
279    }
280
281    // create scan instance
282    TScan scan = new TScan();
283    List<TColumn> columns = new ArrayList<>(1);
284    TColumn column = new TColumn();
285    column.setFamily(familyAname);
286    column.setQualifier(qualifierAname);
287    columns.add(column);
288    scan.setColumns(columns);
289    scan.setStartRow(Bytes.toBytes("testGetScannerResults"));
290
291    // get 5 rows and check the returned results
292    scan.setStopRow(Bytes.toBytes("testGetScannerResults05"));
293    TAuthorization tauth = new TAuthorization();
294    List<String> labels = new ArrayList<>(2);
295    labels.add(SECRET);
296    labels.add(PRIVATE);
297    tauth.setLabels(labels);
298    scan.setAuthorizations(tauth);
299    List<TResult> results = handler.getScannerResults(table, scan, 5);
300    assertEquals(4, results.size());
301    for (int i = 0; i < 4; i++) {
302      if (i < 3) {
303        assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)),
304          results.get(i).getRow());
305      } else if (i == 3) {
306        continue;
307      } else {
308        assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i + 1, (byte) 2)),
309          results.get(i).getRow());
310      }
311    }
312  }
313
314  @Test
315  public void testGetsWithLabels() throws Exception {
316    ThriftHBaseServiceHandler handler = createHandler();
317    byte[] rowName = Bytes.toBytes("testPutGet");
318    ByteBuffer table = wrap(tableAname);
319
320    List<TColumnValue> columnValues = new ArrayList<>(2);
321    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
322    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
323    TPut put = new TPut(wrap(rowName), columnValues);
324
325    put.setColumnValues(columnValues);
326    put.setCellVisibility(new TCellVisibility()
327      .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
328    handler.put(table, put);
329    TGet get = new TGet(wrap(rowName));
330    TAuthorization tauth = new TAuthorization();
331    List<String> labels = new ArrayList<>(2);
332    labels.add(SECRET);
333    labels.add(PRIVATE);
334    tauth.setLabels(labels);
335    get.setAuthorizations(tauth);
336    TResult result = handler.get(table, get);
337    assertArrayEquals(rowName, result.getRow());
338    List<TColumnValue> returnedColumnValues = result.getColumnValues();
339    assertTColumnValuesEqual(columnValues, returnedColumnValues);
340  }
341
342  @Test
343  public void testIncrementWithTags() throws Exception {
344    ThriftHBaseServiceHandler handler = createHandler();
345    byte[] rowName = Bytes.toBytes("testIncrementWithTags");
346    ByteBuffer table = wrap(tableAname);
347
348    List<TColumnValue> columnValues = new ArrayList<>(1);
349    columnValues
350      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
351    TPut put = new TPut(wrap(rowName), columnValues);
352    put.setColumnValues(columnValues);
353    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
354    handler.put(table, put);
355
356    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
357    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
358    TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
359    increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
360    handler.increment(table, increment);
361
362    TGet get = new TGet(wrap(rowName));
363    TAuthorization tauth = new TAuthorization();
364    List<String> labels = new ArrayList<>(1);
365    labels.add(SECRET);
366    tauth.setLabels(labels);
367    get.setAuthorizations(tauth);
368    TResult result = handler.get(table, get);
369
370    assertArrayEquals(rowName, result.getRow());
371    assertEquals(1, result.getColumnValuesSize());
372    TColumnValue columnValue = result.getColumnValues().get(0);
373    assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
374  }
375
376  @Test
377  public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
378    ThriftHBaseServiceHandler handler = createHandler();
379    byte[] rowName = Bytes.toBytes("testIncrementWithTagsWithNotMatchLabels");
380    ByteBuffer table = wrap(tableAname);
381
382    List<TColumnValue> columnValues = new ArrayList<>(1);
383    columnValues
384      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
385    TPut put = new TPut(wrap(rowName), columnValues);
386    put.setColumnValues(columnValues);
387    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
388    handler.put(table, put);
389
390    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
391    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
392    TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
393    increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
394    handler.increment(table, increment);
395
396    TGet get = new TGet(wrap(rowName));
397    TAuthorization tauth = new TAuthorization();
398    List<String> labels = new ArrayList<>(1);
399    labels.add(PUBLIC);
400    tauth.setLabels(labels);
401    get.setAuthorizations(tauth);
402    TResult result = handler.get(table, get);
403    assertNull(result.getRow());
404  }
405
406  @Test
407  public void testAppend() throws Exception {
408    ThriftHBaseServiceHandler handler = createHandler();
409    byte[] rowName = Bytes.toBytes("testAppend");
410    ByteBuffer table = wrap(tableAname);
411    byte[] v1 = Bytes.toBytes(1L);
412    byte[] v2 = Bytes.toBytes(5L);
413    List<TColumnValue> columnValues = new ArrayList<>(1);
414    columnValues
415      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
416    TPut put = new TPut(wrap(rowName), columnValues);
417    put.setColumnValues(columnValues);
418    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
419    handler.put(table, put);
420
421    List<TColumnValue> appendColumns = new ArrayList<>(1);
422    appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(v2)));
423    TAppend append = new TAppend(wrap(rowName), appendColumns);
424    append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
425    handler.append(table, append);
426
427    TGet get = new TGet(wrap(rowName));
428    TAuthorization tauth = new TAuthorization();
429    List<String> labels = new ArrayList<>(1);
430    labels.add(SECRET);
431    tauth.setLabels(labels);
432    get.setAuthorizations(tauth);
433    TResult result = handler.get(table, get);
434
435    assertArrayEquals(rowName, result.getRow());
436    assertEquals(1, result.getColumnValuesSize());
437    TColumnValue columnValue = result.getColumnValues().get(0);
438    assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
439  }
440
441  /**
442   * Padding numbers to make comparison of sort order easier in a for loop The number to pad.
443   * @param n   The number to pad.
444   * @param pad The length to pad up to.
445   * @return The padded number as a string.
446   */
447  private String pad(int n, byte pad) {
448    String res = Integer.toString(n);
449    while (res.length() < pad) {
450      res = "0" + res;
451    }
452    return res;
453  }
454}