001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import java.io.IOException;
021import org.apache.hadoop.hbase.client.Delete;
022import org.apache.hadoop.hbase.client.Durability;
023import org.apache.hadoop.hbase.client.Get;
024import org.apache.hadoop.hbase.client.Put;
025import org.apache.hadoop.hbase.client.Result;
026import org.apache.hadoop.hbase.client.ResultScanner;
027import org.apache.hadoop.hbase.client.Scan;
028import org.apache.hadoop.hbase.client.Table;
029import org.apache.hadoop.hbase.util.Bytes;
030import org.junit.Assert;
031
032/**
033 * Tests user specifiable time stamps putting, getting and scanning. Also tests same in presence of
034 * deletes. Test cores are written so can be run against an HRegion and against an HTable: i.e. both
035 * local and remote.
036 */
037public class TimestampTestBase {
038  private static final long T0 = 10L;
039  private static final long T1 = 100L;
040  private static final long T2 = 200L;
041
042  public static final byte[] FAMILY_NAME = Bytes.toBytes("colfamily11");
043  private static final byte[] QUALIFIER_NAME = Bytes.toBytes("contents");
044
045  private static final byte[] ROW = Bytes.toBytes("row");
046
047  interface FlushCache {
048    void flushcache() throws IOException;
049  }
050
051  /*
052   * Run test that delete works according to description in <a
053   * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>.
054   */
055  public static void doTestDelete(final Table table, FlushCache flusher) throws IOException {
056    // Add values at various timestamps (Values are timestampes as bytes).
057    put(table, T0);
058    put(table, T1);
059    put(table, T2);
060    put(table);
061    // Verify that returned versions match passed timestamps.
062    assertVersions(table, new long[] { HConstants.LATEST_TIMESTAMP, T2, T1 });
063
064    // If I delete w/o specifying a timestamp, this means I'm deleting the latest.
065    delete(table);
066    // Verify that I get back T2 through T1 -- that the latest version has been deleted.
067    assertVersions(table, new long[] { T2, T1, T0 });
068
069    // Flush everything out to disk and then retry
070    flusher.flushcache();
071    assertVersions(table, new long[] { T2, T1, T0 });
072
073    // Now add, back a latest so I can test remove other than the latest.
074    put(table);
075    assertVersions(table, new long[] { HConstants.LATEST_TIMESTAMP, T2, T1 });
076    delete(table, T2);
077    assertVersions(table, new long[] { HConstants.LATEST_TIMESTAMP, T1, T0 });
078    // Flush everything out to disk and then retry
079    flusher.flushcache();
080    assertVersions(table, new long[] { HConstants.LATEST_TIMESTAMP, T1, T0 });
081
082    // Now try deleting all from T2 back inclusive (We first need to add T2
083    // back into the mix and to make things a little interesting, delete and then readd T1.
084    put(table, T2);
085    delete(table, T1);
086    put(table, T1);
087
088    Delete delete = new Delete(ROW);
089    delete.addColumns(FAMILY_NAME, QUALIFIER_NAME, T2);
090    table.delete(delete);
091
092    // Should only be current value in set. Assert this is so
093    assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
094
095    // Flush everything out to disk and then redo above tests
096    flusher.flushcache();
097    assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
098  }
099
100  private static void assertOnlyLatest(final Table incommon, final long currentTime)
101    throws IOException {
102    Get get = null;
103    get = new Get(ROW);
104    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
105    get.setMaxVersions(3);
106    Result result = incommon.get(get);
107    Assert.assertEquals(1, result.size());
108    long time = Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]));
109    Assert.assertEquals(time, currentTime);
110  }
111
112  /*
113   * Assert that returned versions match passed in timestamps and that results are returned in the
114   * right order. Assert that values when converted to longs match the corresponding passed
115   * timestamp.
116   */
117  public static void assertVersions(final Table incommon, final long[] tss) throws IOException {
118    // Assert that 'latest' is what we expect.
119    Get get = null;
120    get = new Get(ROW);
121    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
122    Result r = incommon.get(get);
123    byte[] bytes = r.getValue(FAMILY_NAME, QUALIFIER_NAME);
124    long t = Bytes.toLong(bytes);
125    Assert.assertEquals(tss[0], t);
126
127    // Now assert that if we ask for multiple versions, that they come out in
128    // order.
129    get = new Get(ROW);
130    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
131    get.setMaxVersions(tss.length);
132    Result result = incommon.get(get);
133    Cell[] kvs = result.rawCells();
134    Assert.assertEquals(kvs.length, tss.length);
135    for (int i = 0; i < kvs.length; i++) {
136      t = Bytes.toLong(CellUtil.cloneValue(kvs[i]));
137      Assert.assertEquals(tss[i], t);
138    }
139
140    // Determine highest stamp to set as next max stamp
141    long maxStamp = kvs[0].getTimestamp();
142
143    // Specify a timestamp get multiple versions.
144    get = new Get(ROW);
145    get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
146    get.setTimeRange(0, maxStamp);
147    get.setMaxVersions(kvs.length - 1);
148    result = incommon.get(get);
149    kvs = result.rawCells();
150    Assert.assertEquals(kvs.length, tss.length - 1);
151    for (int i = 1; i < kvs.length; i++) {
152      t = Bytes.toLong(CellUtil.cloneValue(kvs[i - 1]));
153      Assert.assertEquals(tss[i], t);
154    }
155
156    // Test scanner returns expected version
157    assertScanContentTimestamp(incommon, tss[0]);
158  }
159
160  /*
161   * Run test scanning different timestamps.
162   */
163  public static void doTestTimestampScanning(final Table incommon, final FlushCache flusher)
164    throws IOException {
165    // Add a couple of values for three different timestamps.
166    put(incommon, T0);
167    put(incommon, T1);
168    put(incommon, HConstants.LATEST_TIMESTAMP);
169    // Get count of latest items.
170    int count = assertScanContentTimestamp(incommon, HConstants.LATEST_TIMESTAMP);
171    // Assert I get same count when I scan at each timestamp.
172    Assert.assertEquals(count, assertScanContentTimestamp(incommon, T0));
173    Assert.assertEquals(count, assertScanContentTimestamp(incommon, T1));
174    // Flush everything out to disk and then retry
175    flusher.flushcache();
176    Assert.assertEquals(count, assertScanContentTimestamp(incommon, T0));
177    Assert.assertEquals(count, assertScanContentTimestamp(incommon, T1));
178  }
179
180  /*
181   * Assert that the scan returns only values < timestamp.
182   * @return Count of items scanned.
183   */
184  public static int assertScanContentTimestamp(final Table in, final long ts) throws IOException {
185    Scan scan = new Scan(HConstants.EMPTY_START_ROW);
186    scan.addFamily(FAMILY_NAME);
187    scan.setTimeRange(0, ts);
188    ResultScanner scanner = in.getScanner(scan);
189    int count = 0;
190    try {
191      // TODO FIX
192      // HStoreKey key = new HStoreKey();
193      // TreeMap<byte [], Cell>value =
194      // new TreeMap<byte [], Cell>(Bytes.BYTES_COMPARATOR);
195      // while (scanner.next(key, value)) {
196      // assertTrue(key.getTimestamp() <= ts);
197      // // Content matches the key or HConstants.LATEST_TIMESTAMP.
198      // // (Key does not match content if we 'put' with LATEST_TIMESTAMP).
199      // long l = Bytes.toLong(value.get(COLUMN).getValue());
200      // assertTrue(key.getTimestamp() == l ||
201      // HConstants.LATEST_TIMESTAMP == l);
202      // count++;
203      // value.clear();
204      // }
205    } finally {
206      scanner.close();
207    }
208    return count;
209  }
210
211  public static void put(final Table loader, final long ts) throws IOException {
212    put(loader, Bytes.toBytes(ts), ts);
213  }
214
215  public static void put(final Table loader) throws IOException {
216    long ts = HConstants.LATEST_TIMESTAMP;
217    put(loader, Bytes.toBytes(ts), ts);
218  }
219
220  /*
221   * Put values.
222   */
223  public static void put(final Table loader, final byte[] bytes, final long ts) throws IOException {
224    Put put = new Put(ROW, ts);
225    put.setDurability(Durability.SKIP_WAL);
226    put.addColumn(FAMILY_NAME, QUALIFIER_NAME, bytes);
227    loader.put(put);
228  }
229
230  public static void delete(final Table loader) throws IOException {
231    delete(loader, null);
232  }
233
234  public static void delete(final Table loader, final byte[] column) throws IOException {
235    delete(loader, column, HConstants.LATEST_TIMESTAMP);
236  }
237
238  public static void delete(final Table loader, final long ts) throws IOException {
239    delete(loader, null, ts);
240  }
241
242  public static void delete(final Table loader, final byte[] column, final long ts)
243    throws IOException {
244    Delete delete = ts == HConstants.LATEST_TIMESTAMP ? new Delete(ROW) : new Delete(ROW, ts);
245    delete.addColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
246    loader.delete(delete);
247  }
248
249  public static Result get(final Table loader) throws IOException {
250    return loader.get(new Get(ROW));
251  }
252}