Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rest;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.ByteArrayInputStream;
026import java.io.IOException;
027import java.io.StringWriter;
028import java.util.ArrayList;
029import java.util.Iterator;
030import java.util.List;
031import java.util.Random;
032import java.util.concurrent.ThreadLocalRandom;
033import javax.xml.bind.JAXBContext;
034import javax.xml.bind.JAXBException;
035import javax.xml.bind.Marshaller;
036import javax.xml.bind.Unmarshaller;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.CellUtil;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtil;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.client.Admin;
043import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
045import org.apache.hadoop.hbase.client.Connection;
046import org.apache.hadoop.hbase.client.ConnectionFactory;
047import org.apache.hadoop.hbase.client.Durability;
048import org.apache.hadoop.hbase.client.Put;
049import org.apache.hadoop.hbase.client.Table;
050import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
051import org.apache.hadoop.hbase.rest.client.Client;
052import org.apache.hadoop.hbase.rest.client.Cluster;
053import org.apache.hadoop.hbase.rest.client.Response;
054import org.apache.hadoop.hbase.rest.model.CellModel;
055import org.apache.hadoop.hbase.rest.model.CellSetModel;
056import org.apache.hadoop.hbase.rest.model.RowModel;
057import org.apache.hadoop.hbase.rest.model.ScannerModel;
058import org.apache.hadoop.hbase.testclassification.MediumTests;
059import org.apache.hadoop.hbase.testclassification.RestTests;
060import org.apache.hadoop.hbase.util.Bytes;
061import org.apache.http.Header;
062import org.junit.AfterClass;
063import org.junit.BeforeClass;
064import org.junit.ClassRule;
065import org.junit.Test;
066import org.junit.experimental.categories.Category;
067import org.slf4j.Logger;
068import org.slf4j.LoggerFactory;
069
070@Category({ RestTests.class, MediumTests.class })
071public class TestScannerResource {
072
073  @ClassRule
074  public static final HBaseClassTestRule CLASS_RULE =
075    HBaseClassTestRule.forClass(TestScannerResource.class);
076
077  private static final Logger LOG = LoggerFactory.getLogger(TestScannerResource.class);
078  private static final TableName TABLE = TableName.valueOf("TestScannerResource");
079  private static final TableName TABLE_TO_BE_DISABLED = TableName.valueOf("ScannerResourceDisable");
080  private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
081  private static final String CFA = "a";
082  private static final String CFB = "b";
083  private static final String COLUMN_1 = CFA + ":1";
084  private static final String COLUMN_2 = CFB + ":2";
085
086  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
087  private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility();
088  private static Client client;
089  private static JAXBContext context;
090  private static Marshaller marshaller;
091  private static Unmarshaller unmarshaller;
092  private static int expectedRows1;
093  private static int expectedRows2;
094  private static Configuration conf;
095
096  static int insertData(Configuration conf, TableName tableName, String column, double prob)
097    throws IOException {
098    Random rng = ThreadLocalRandom.current();
099    byte[] k = new byte[3];
100    byte[][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(column));
101    List<Put> puts = new ArrayList<>();
102    for (byte b1 = 'a'; b1 < 'z'; b1++) {
103      for (byte b2 = 'a'; b2 < 'z'; b2++) {
104        for (byte b3 = 'a'; b3 < 'z'; b3++) {
105          if (rng.nextDouble() < prob) {
106            k[0] = b1;
107            k[1] = b2;
108            k[2] = b3;
109            Put put = new Put(k);
110            put.setDurability(Durability.SKIP_WAL);
111            put.addColumn(famAndQf[0], famAndQf[1], k);
112            puts.add(put);
113          }
114        }
115      }
116    }
117    try (Connection conn = ConnectionFactory.createConnection(conf);
118      Table table = conn.getTable(tableName)) {
119      table.put(puts);
120    }
121    return puts.size();
122  }
123
124  static int countCellSet(CellSetModel model) {
125    int count = 0;
126    Iterator<RowModel> rows = model.getRows().iterator();
127    while (rows.hasNext()) {
128      RowModel row = rows.next();
129      Iterator<CellModel> cells = row.getCells().iterator();
130      while (cells.hasNext()) {
131        cells.next();
132        count++;
133      }
134    }
135    return count;
136  }
137
138  private static int fullTableScan(ScannerModel model) throws IOException {
139    model.setBatch(100);
140    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
141      model.createProtobufOutput());
142    assertEquals(201, response.getCode());
143    String scannerURI = response.getLocation();
144    assertNotNull(scannerURI);
145    int count = 0;
146    while (true) {
147      response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
148      assertTrue(response.getCode() == 200 || response.getCode() == 204);
149      if (response.getCode() == 200) {
150        assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
151        CellSetModel cellSet = new CellSetModel();
152        cellSet.getObjectFromMessage(response.getBody());
153        Iterator<RowModel> rows = cellSet.getRows().iterator();
154        while (rows.hasNext()) {
155          RowModel row = rows.next();
156          Iterator<CellModel> cells = row.getCells().iterator();
157          while (cells.hasNext()) {
158            cells.next();
159            count++;
160          }
161        }
162      } else {
163        break;
164      }
165    }
166    // delete the scanner
167    response = client.delete(scannerURI);
168    assertEquals(200, response.getCode());
169    return count;
170  }
171
172  @BeforeClass
173  public static void setUpBeforeClass() throws Exception {
174    conf = TEST_UTIL.getConfiguration();
175    TEST_UTIL.startMiniCluster();
176    REST_TEST_UTIL.startServletContainer(conf);
177    client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()));
178    context = JAXBContext.newInstance(CellModel.class, CellSetModel.class, RowModel.class,
179      ScannerModel.class);
180    marshaller = context.createMarshaller();
181    unmarshaller = context.createUnmarshaller();
182    Admin admin = TEST_UTIL.getAdmin();
183    if (admin.tableExists(TABLE)) {
184      return;
185    }
186    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(TABLE);
187    ColumnFamilyDescriptor columnFamilyDescriptor =
188      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFA)).build();
189    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
190    columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFB)).build();
191    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
192
193    admin.createTable(tableDescriptorBuilder.build());
194    expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
195    expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
196
197    tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(TABLE_TO_BE_DISABLED);
198    columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFA)).build();
199    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
200    columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFB)).build();
201    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
202
203    admin.createTable(tableDescriptorBuilder.build());
204  }
205
206  @AfterClass
207  public static void tearDownAfterClass() throws Exception {
208    REST_TEST_UTIL.shutdownServletContainer();
209    TEST_UTIL.shutdownMiniCluster();
210  }
211
212  @Test
213  public void testSimpleScannerXML() throws IOException, JAXBException {
214    final int BATCH_SIZE = 5;
215    // new scanner
216    ScannerModel model = new ScannerModel();
217    model.setBatch(BATCH_SIZE);
218    model.addColumn(Bytes.toBytes(COLUMN_1));
219    StringWriter writer = new StringWriter();
220    marshaller.marshal(model, writer);
221    byte[] body = Bytes.toBytes(writer.toString());
222
223    // test put operation is forbidden in read-only mode
224    conf.set("hbase.rest.readonly", "true");
225    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
226    assertEquals(403, response.getCode());
227    String scannerURI = response.getLocation();
228    assertNull(scannerURI);
229
230    // recall previous put operation with read-only off
231    conf.set("hbase.rest.readonly", "false");
232    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
233    assertEquals(201, response.getCode());
234    scannerURI = response.getLocation();
235    assertNotNull(scannerURI);
236
237    // get a cell set
238    response = client.get(scannerURI, Constants.MIMETYPE_XML);
239    assertEquals(200, response.getCode());
240    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
241    CellSetModel cellSet =
242      (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
243    // confirm batch size conformance
244    assertEquals(BATCH_SIZE, countCellSet(cellSet));
245
246    // test delete scanner operation is forbidden in read-only mode
247    conf.set("hbase.rest.readonly", "true");
248    response = client.delete(scannerURI);
249    assertEquals(403, response.getCode());
250
251    // recall previous delete scanner operation with read-only off
252    conf.set("hbase.rest.readonly", "false");
253    response = client.delete(scannerURI);
254    assertEquals(200, response.getCode());
255  }
256
257  @Test
258  public void testSimpleScannerPB() throws IOException {
259    final int BATCH_SIZE = 10;
260    // new scanner
261    ScannerModel model = new ScannerModel();
262    model.setBatch(BATCH_SIZE);
263    model.addColumn(Bytes.toBytes(COLUMN_1));
264
265    // test put operation is forbidden in read-only mode
266    conf.set("hbase.rest.readonly", "true");
267    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
268      model.createProtobufOutput());
269    assertEquals(403, response.getCode());
270    String scannerURI = response.getLocation();
271    assertNull(scannerURI);
272
273    // recall previous put operation with read-only off
274    conf.set("hbase.rest.readonly", "false");
275    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
276      model.createProtobufOutput());
277    assertEquals(201, response.getCode());
278    scannerURI = response.getLocation();
279    assertNotNull(scannerURI);
280
281    // get a cell set
282    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
283    assertEquals(200, response.getCode());
284    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
285    CellSetModel cellSet = new CellSetModel();
286    cellSet.getObjectFromMessage(response.getBody());
287    // confirm batch size conformance
288    assertEquals(BATCH_SIZE, countCellSet(cellSet));
289
290    // test delete scanner operation is forbidden in read-only mode
291    conf.set("hbase.rest.readonly", "true");
292    response = client.delete(scannerURI);
293    assertEquals(403, response.getCode());
294
295    // recall previous delete scanner operation with read-only off
296    conf.set("hbase.rest.readonly", "false");
297    response = client.delete(scannerURI);
298    assertEquals(200, response.getCode());
299  }
300
301  @Test
302  public void testSimpleScannerBinary() throws IOException {
303    // new scanner
304    ScannerModel model = new ScannerModel();
305    model.setBatch(1);
306    model.addColumn(Bytes.toBytes(COLUMN_1));
307
308    // test put operation is forbidden in read-only mode
309    conf.set("hbase.rest.readonly", "true");
310    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
311      model.createProtobufOutput());
312    assertEquals(403, response.getCode());
313    String scannerURI = response.getLocation();
314    assertNull(scannerURI);
315
316    // recall previous put operation with read-only off
317    conf.set("hbase.rest.readonly", "false");
318    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
319      model.createProtobufOutput());
320    assertEquals(201, response.getCode());
321    scannerURI = response.getLocation();
322    assertNotNull(scannerURI);
323
324    // get a cell
325    response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
326    assertEquals(200, response.getCode());
327    assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
328    // verify that data was returned
329    assertTrue(response.getBody().length > 0);
330    // verify that the expected X-headers are present
331    boolean foundRowHeader = false, foundColumnHeader = false, foundTimestampHeader = false;
332    for (Header header : response.getHeaders()) {
333      if (header.getName().equals("X-Row")) {
334        foundRowHeader = true;
335      } else if (header.getName().equals("X-Column")) {
336        foundColumnHeader = true;
337      } else if (header.getName().equals("X-Timestamp")) {
338        foundTimestampHeader = true;
339      }
340    }
341    assertTrue(foundRowHeader);
342    assertTrue(foundColumnHeader);
343    assertTrue(foundTimestampHeader);
344
345    // test delete scanner operation is forbidden in read-only mode
346    conf.set("hbase.rest.readonly", "true");
347    response = client.delete(scannerURI);
348    assertEquals(403, response.getCode());
349
350    // recall previous delete scanner operation with read-only off
351    conf.set("hbase.rest.readonly", "false");
352    response = client.delete(scannerURI);
353    assertEquals(200, response.getCode());
354  }
355
356  @Test
357  public void testFullTableScan() throws IOException {
358    ScannerModel model = new ScannerModel();
359    model.addColumn(Bytes.toBytes(COLUMN_1));
360    assertEquals(expectedRows1, fullTableScan(model));
361
362    model = new ScannerModel();
363    model.addColumn(Bytes.toBytes(COLUMN_2));
364    assertEquals(expectedRows2, fullTableScan(model));
365  }
366
367  @Test
368  public void testTableDoesNotExist() throws IOException, JAXBException {
369    ScannerModel model = new ScannerModel();
370    StringWriter writer = new StringWriter();
371    marshaller.marshal(model, writer);
372    byte[] body = Bytes.toBytes(writer.toString());
373    Response response =
374      client.put("/" + NONEXISTENT_TABLE + "/scanner", Constants.MIMETYPE_XML, body);
375    String scannerURI = response.getLocation();
376    assertNotNull(scannerURI);
377    response = client.get(scannerURI, Constants.MIMETYPE_XML);
378    assertEquals(404, response.getCode());
379  }
380
381  @Test
382  public void testTableScanWithTableDisable() throws IOException {
383    TEST_UTIL.getAdmin().disableTable(TABLE_TO_BE_DISABLED);
384    ScannerModel model = new ScannerModel();
385    model.addColumn(Bytes.toBytes(COLUMN_1));
386    model.setCaching(1);
387    Response response = client.put("/" + TABLE_TO_BE_DISABLED + "/scanner",
388      Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
389    // we will see the exception when we actually want to get the result.
390    assertEquals(201, response.getCode());
391    String scannerURI = response.getLocation();
392    assertNotNull(scannerURI);
393    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
394    assertEquals(410, response.getCode());
395  }
396
397  @Test
398  public void deleteNonExistent() throws IOException {
399    Response response = client.delete("/" + TABLE + "/scanner/NONEXISTENT_SCAN");
400    assertEquals(404, response.getCode());
401  }
402}