View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  
23  import org.apache.hadoop.hbase.client.Delete;
24  import org.apache.hadoop.hbase.client.Get;
25  import org.apache.hadoop.hbase.client.Put;
26  import org.apache.hadoop.hbase.client.Result;
27  import org.apache.hadoop.hbase.client.ResultScanner;
28  import org.apache.hadoop.hbase.client.Scan;
29  import org.apache.hadoop.hbase.client.Table;
30  import org.apache.hadoop.hbase.client.Durability;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.junit.Assert;
33  
34  /**
35   * Tests user specifiable time stamps putting, getting and scanning.  Also
36   * tests same in presence of deletes.  Test cores are written so can be
37   * run against an HRegion and against an HTable: i.e. both local and remote.
38   */
39  public class TimestampTestBase {
40    private static final long T0 = 10L;
41    private static final long T1 = 100L;
42    private static final long T2 = 200L;
43  
44    public static final byte [] FAMILY_NAME = Bytes.toBytes("colfamily11");
45    private static final byte [] QUALIFIER_NAME = Bytes.toBytes("contents");
46  
47    private static final byte [] ROW = Bytes.toBytes("row");
48  
49    interface FlushCache {
50      void flushcache() throws IOException;
51    }
52  
53    /*
54     * Run test that delete works according to description in <a
55     * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>.
56     * @param incommon
57     * @param flusher
58     * @throws IOException
59     */
60    public static void doTestDelete(final Table table, FlushCache flusher)
61    throws IOException {
62      // Add values at various timestamps (Values are timestampes as bytes).
63      put(table, T0);
64      put(table, T1);
65      put(table, T2);
66      put(table);
67      // Verify that returned versions match passed timestamps.
68      assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
69  
70      // If I delete w/o specifying a timestamp, this means I'm deleting the latest.
71      delete(table);
72      // Verify that I get back T2 through T1 -- that the latest version has been deleted.
73      assertVersions(table, new long [] {T2, T1, T0});
74  
75      // Flush everything out to disk and then retry
76      flusher.flushcache();
77      assertVersions(table, new long [] {T2, T1, T0});
78  
79      // Now add, back a latest so I can test remove other than the latest.
80      put(table);
81      assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
82      delete(table, T2);
83      assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
84      // Flush everything out to disk and then retry
85      flusher.flushcache();
86      assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
87  
88      // Now try deleting all from T2 back inclusive (We first need to add T2
89      // back into the mix and to make things a little interesting, delete and then readd T1.
90      put(table, T2);
91      delete(table, T1);
92      put(table, T1);
93  
94      Delete delete = new Delete(ROW);
95      delete.addColumns(FAMILY_NAME, QUALIFIER_NAME, T2);
96      table.delete(delete);
97  
98      // Should only be current value in set.  Assert this is so
99      assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
100 
101     // Flush everything out to disk and then redo above tests
102     flusher.flushcache();
103     assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
104   }
105 
106   private static void assertOnlyLatest(final Table incommon, final long currentTime)
107   throws IOException {
108     Get get = null;
109     get = new Get(ROW);
110     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
111     get.setMaxVersions(3);
112     Result result = incommon.get(get);
113     Assert.assertEquals(1, result.size());
114     long time = Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]));
115     Assert.assertEquals(time, currentTime);
116   }
117 
118   /*
119    * Assert that returned versions match passed in timestamps and that results
120    * are returned in the right order.  Assert that values when converted to
121    * longs match the corresponding passed timestamp.
122    * @param r
123    * @param tss
124    * @throws IOException
125    */
126   public static void assertVersions(final Table incommon, final long [] tss)
127   throws IOException {
128     // Assert that 'latest' is what we expect.
129     Get get = null;
130     get = new Get(ROW);
131     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
132     Result r = incommon.get(get);
133     byte [] bytes = r.getValue(FAMILY_NAME, QUALIFIER_NAME);
134     long t = Bytes.toLong(bytes);
135     Assert.assertEquals(tss[0], t);
136 
137     // Now assert that if we ask for multiple versions, that they come out in
138     // order.
139     get = new Get(ROW);
140     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
141     get.setMaxVersions(tss.length);
142     Result result = incommon.get(get);
143     Cell [] kvs = result.rawCells();
144     Assert.assertEquals(kvs.length, tss.length);
145     for(int i=0;i<kvs.length;i++) {
146       t = Bytes.toLong(CellUtil.cloneValue(kvs[i]));
147       Assert.assertEquals(tss[i], t);
148     }
149 
150     // Determine highest stamp to set as next max stamp
151     long maxStamp = kvs[0].getTimestamp();
152 
153     // Specify a timestamp get multiple versions.
154     get = new Get(ROW);
155     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
156     get.setTimeRange(0, maxStamp);
157     get.setMaxVersions(kvs.length - 1);
158     result = incommon.get(get);
159     kvs = result.rawCells();
160     Assert.assertEquals(kvs.length, tss.length - 1);
161     for(int i=1;i<kvs.length;i++) {
162       t = Bytes.toLong(CellUtil.cloneValue(kvs[i-1]));
163       Assert.assertEquals(tss[i], t);
164     }
165 
166     // Test scanner returns expected version
167     assertScanContentTimestamp(incommon, tss[0]);
168   }
169 
170   /*
171    * Run test scanning different timestamps.
172    * @param incommon
173    * @param flusher
174    * @throws IOException
175    */
176   public static void doTestTimestampScanning(final Table incommon,
177     final FlushCache flusher)
178   throws IOException {
179     // Add a couple of values for three different timestamps.
180     put(incommon, T0);
181     put(incommon, T1);
182     put(incommon, HConstants.LATEST_TIMESTAMP);
183     // Get count of latest items.
184     int count = assertScanContentTimestamp(incommon,
185       HConstants.LATEST_TIMESTAMP);
186     // Assert I get same count when I scan at each timestamp.
187     Assert.assertEquals(count, assertScanContentTimestamp(incommon, T0));
188     Assert.assertEquals(count, assertScanContentTimestamp(incommon, T1));
189     // Flush everything out to disk and then retry
190     flusher.flushcache();
191     Assert.assertEquals(count, assertScanContentTimestamp(incommon, T0));
192     Assert.assertEquals(count, assertScanContentTimestamp(incommon, T1));
193   }
194 
195   /*
196    * Assert that the scan returns only values < timestamp.
197    * @param r
198    * @param ts
199    * @return Count of items scanned.
200    * @throws IOException
201    */
202   public static int assertScanContentTimestamp(final Table in, final long ts)
203   throws IOException {
204     Scan scan = new Scan(HConstants.EMPTY_START_ROW);
205     scan.addFamily(FAMILY_NAME);
206     scan.setTimeRange(0, ts);
207     ResultScanner scanner = in.getScanner(scan);
208     int count = 0;
209     try {
210       // TODO FIX
211 //      HStoreKey key = new HStoreKey();
212 //      TreeMap<byte [], Cell>value =
213 //        new TreeMap<byte [], Cell>(Bytes.BYTES_COMPARATOR);
214 //      while (scanner.next(key, value)) {
215 //        assertTrue(key.getTimestamp() <= ts);
216 //        // Content matches the key or HConstants.LATEST_TIMESTAMP.
217 //        // (Key does not match content if we 'put' with LATEST_TIMESTAMP).
218 //        long l = Bytes.toLong(value.get(COLUMN).getValue());
219 //        assertTrue(key.getTimestamp() == l ||
220 //          HConstants.LATEST_TIMESTAMP == l);
221 //        count++;
222 //        value.clear();
223 //      }
224     } finally {
225       scanner.close();
226     }
227     return count;
228   }
229 
230   public static void put(final Table loader, final long ts)
231   throws IOException {
232     put(loader, Bytes.toBytes(ts), ts);
233   }
234 
235   public static void put(final Table loader)
236   throws IOException {
237     long ts = HConstants.LATEST_TIMESTAMP;
238     put(loader, Bytes.toBytes(ts), ts);
239   }
240 
241   /*
242    * Put values.
243    * @param loader
244    * @param bytes
245    * @param ts
246    * @throws IOException
247    */
248   public static void put(final Table loader, final byte [] bytes,
249     final long ts)
250   throws IOException {
251     Put put = new Put(ROW, ts);
252     put.setDurability(Durability.SKIP_WAL);
253     put.addColumn(FAMILY_NAME, QUALIFIER_NAME, bytes);
254     loader.put(put);
255   }
256 
257   public static void delete(final Table loader) throws IOException {
258     delete(loader, null);
259   }
260 
261   public static void delete(final Table loader, final byte [] column)
262   throws IOException {
263     delete(loader, column, HConstants.LATEST_TIMESTAMP);
264   }
265 
266   public static void delete(final Table loader, final long ts)
267   throws IOException {
268     delete(loader, null, ts);
269   }
270 
271   public static void delete(final Table loader, final byte [] column,
272       final long ts)
273   throws IOException {
274     Delete delete = ts == HConstants.LATEST_TIMESTAMP?
275       new Delete(ROW): new Delete(ROW, ts);
276     delete.addColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
277     loader.delete(delete);
278   }
279 
280   public static Result get(final Table loader) throws IOException {
281     return loader.get(new Get(ROW));
282   }
283 }