1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.*;
21
22 import java.io.IOException;
23 import java.util.Collections;
24 import java.util.Comparator;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.concurrent.CountDownLatch;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.commons.logging.impl.Log4JLogger;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.TableName;
40 import org.apache.hadoop.hbase.TableNotFoundException;
41 import org.apache.hadoop.hbase.client.Admin;
42 import org.apache.hadoop.hbase.client.HTable;
43 import org.apache.hadoop.hbase.client.ScannerCallable;
44 import org.apache.hadoop.hbase.ipc.RpcClient;
45 import org.apache.hadoop.hbase.ipc.RpcServer;
46 import org.apache.hadoop.hbase.master.HMaster;
47 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
48 import org.apache.hadoop.hbase.mob.MobConstants;
49 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
50 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
51 import org.apache.hadoop.hbase.testclassification.LargeTests;
52 import org.apache.hadoop.hbase.util.Bytes;
53 import org.apache.hadoop.hbase.util.FSUtils;
54 import org.apache.log4j.Level;
55 import org.junit.After;
56 import org.junit.AfterClass;
57 import org.junit.Before;
58 import org.junit.BeforeClass;
59 import org.junit.Test;
60 import org.junit.experimental.categories.Category;
61
62
63
64
65
66
67
68
69
70 @Category(LargeTests.class)
71 public class TestMobFlushSnapshotFromClient {
72 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
73 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
74 private static final int NUM_RS = 2;
75 private static final String STRING_TABLE_NAME = "test";
76 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
77 private static final TableName TABLE_NAME =
78 TableName.valueOf(STRING_TABLE_NAME);
79 private final int DEFAULT_NUM_ROWS = 100;
80
81
82
83
84
85 @BeforeClass
86 public static void setupCluster() throws Exception {
87 ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
88 ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
89 setupConf(UTIL.getConfiguration());
90 UTIL.startMiniCluster(NUM_RS);
91 }
92
93 private static void setupConf(Configuration conf) {
94
95 conf.setInt("hbase.regionsever.info.port", -1);
96
97 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
98
99
100 conf.setInt("hbase.hstore.compaction.min", 10);
101 conf.setInt("hbase.hstore.compactionThreshold", 10);
102
103 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
104
105 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
106 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
107 ConstantSizeRegionSplitPolicy.class.getName());
108 conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0);
109 conf.setInt("hfile.format.version", 3);
110 }
111
112 @Before
113 public void setup() throws Exception {
114 MobSnapshotTestingUtils.createMobTable(UTIL, TABLE_NAME, 1, TEST_FAM);
115 }
116
117 @After
118 public void tearDown() throws Exception {
119 UTIL.deleteTable(TABLE_NAME);
120
121 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
122 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
123 }
124
125 @AfterClass
126 public static void cleanupTest() throws Exception {
127 try {
128 UTIL.shutdownMiniCluster();
129 } catch (Exception e) {
130 LOG.warn("failure shutting down cluster", e);
131 }
132 }
133
134
135
136
137
138 @Test (timeout=300000)
139 public void testFlushTableSnapshot() throws Exception {
140 Admin admin = UTIL.getHBaseAdmin();
141
142 SnapshotTestingUtils.assertNoSnapshots(admin);
143
144
145 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
146 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
147
148 LOG.debug("FS state before snapshot:");
149 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
150 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
151
152
153 String snapshotString = "offlineTableSnapshot";
154 byte[] snapshot = Bytes.toBytes(snapshotString);
155 admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH);
156 LOG.debug("Snapshot completed.");
157
158
159 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
160 snapshot, TABLE_NAME);
161
162
163 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
164 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
165 LOG.debug("FS state after snapshot:");
166 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
167 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
168
169 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
170 admin, fs);
171 }
172
173
174
175
176
177 @Test(timeout=30000)
178 public void testSkipFlushTableSnapshot() throws Exception {
179 Admin admin = UTIL.getHBaseAdmin();
180
181 SnapshotTestingUtils.assertNoSnapshots(admin);
182
183
184 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
185 UTIL.loadTable(table, TEST_FAM);
186
187 LOG.debug("FS state before snapshot:");
188 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
189 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
190
191
192 String snapshotString = "skipFlushTableSnapshot";
193 byte[] snapshot = Bytes.toBytes(snapshotString);
194 admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
195 LOG.debug("Snapshot completed.");
196
197
198 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
199 snapshot, TABLE_NAME);
200
201
202 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
203 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
204 LOG.debug("FS state after snapshot:");
205 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
206 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
207
208 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
209 admin, fs);
210
211 admin.deleteSnapshot(snapshot);
212 snapshots = admin.listSnapshots();
213 SnapshotTestingUtils.assertNoSnapshots(admin);
214 }
215
216
217
218
219
220
221 @Test (timeout=300000)
222 public void testFlushTableSnapshotWithProcedure() throws Exception {
223 Admin admin = UTIL.getHBaseAdmin();
224
225 SnapshotTestingUtils.assertNoSnapshots(admin);
226
227
228 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
229 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
230
231 LOG.debug("FS state before snapshot:");
232 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
233 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
234
235
236 String snapshotString = "offlineTableSnapshot";
237 byte[] snapshot = Bytes.toBytes(snapshotString);
238 Map<String, String> props = new HashMap<String, String>();
239 props.put("table", TABLE_NAME.getNameAsString());
240 admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
241 snapshotString, props);
242
243
244 LOG.debug("Snapshot completed.");
245
246
247 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
248 snapshot, TABLE_NAME);
249
250
251 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
252 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
253 LOG.debug("FS state after snapshot:");
254 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
255 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
256
257 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
258 admin, fs);
259 }
260
261 @Test (timeout=300000)
262 public void testSnapshotFailsOnNonExistantTable() throws Exception {
263 Admin admin = UTIL.getHBaseAdmin();
264
265 SnapshotTestingUtils.assertNoSnapshots(admin);
266 TableName tableName = TableName.valueOf("_not_a_table");
267
268
269 boolean fail = false;
270 do {
271 try {
272 admin.getTableDescriptor(tableName);
273 fail = true;
274 LOG.error("Table:" + tableName + " already exists, checking a new name");
275 tableName = TableName.valueOf(tableName+"!");
276 } catch (TableNotFoundException e) {
277 fail = false;
278 }
279 } while (fail);
280
281
282 try {
283 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
284 fail("Snapshot succeeded even though there is not table.");
285 } catch (SnapshotCreationException e) {
286 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
287 }
288 }
289
290 @Test(timeout = 300000)
291 public void testAsyncFlushSnapshot() throws Exception {
292 Admin admin = UTIL.getHBaseAdmin();
293 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
294 .setTable(TABLE_NAME.getNameAsString())
295 .setType(SnapshotDescription.Type.FLUSH)
296 .build();
297
298
299 admin.takeSnapshotAsync(snapshot);
300
301
302 HMaster master = UTIL.getMiniHBaseCluster().getMaster();
303 SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
304 LOG.info(" === Async Snapshot Completed ===");
305 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
306 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
307
308 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
309 }
310
311 @Test (timeout=300000)
312 public void testSnapshotStateAfterMerge() throws Exception {
313 int numRows = DEFAULT_NUM_ROWS;
314 Admin admin = UTIL.getHBaseAdmin();
315
316 SnapshotTestingUtils.assertNoSnapshots(admin);
317
318 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
319
320
321 String snapshotBeforeMergeName = "snapshotBeforeMerge";
322 admin.snapshot(snapshotBeforeMergeName, TABLE_NAME, SnapshotDescription.Type.FLUSH);
323
324
325 TableName cloneBeforeMergeName = TableName.valueOf("cloneBeforeMerge");
326 admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
327 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneBeforeMergeName);
328
329
330 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
331 Collections.sort(regions, new Comparator<HRegionInfo>() {
332 public int compare(HRegionInfo r1, HRegionInfo r2) {
333 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
334 }
335 });
336
337 int numRegions = admin.getTableRegions(TABLE_NAME).size();
338 int numRegionsAfterMerge = numRegions - 2;
339 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
340 regions.get(2).getEncodedNameAsBytes(), true);
341 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
342 regions.get(6).getEncodedNameAsBytes(), true);
343
344
345 waitRegionsAfterMerge(numRegionsAfterMerge);
346 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
347
348
349 TableName cloneAfterMergeName = TableName.valueOf("cloneAfterMerge");
350 admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
351 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneAfterMergeName);
352
353 MobSnapshotTestingUtils.verifyMobRowCount(UTIL, TABLE_NAME, numRows);
354 MobSnapshotTestingUtils.verifyMobRowCount(UTIL, cloneBeforeMergeName, numRows);
355 MobSnapshotTestingUtils.verifyMobRowCount(UTIL, cloneAfterMergeName, numRows);
356
357
358 UTIL.deleteTable(cloneAfterMergeName);
359 UTIL.deleteTable(cloneBeforeMergeName);
360 }
361
362 @Test (timeout=300000)
363 public void testTakeSnapshotAfterMerge() throws Exception {
364 int numRows = DEFAULT_NUM_ROWS;
365 Admin admin = UTIL.getHBaseAdmin();
366
367 SnapshotTestingUtils.assertNoSnapshots(admin);
368
369 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
370
371
372 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
373 Collections.sort(regions, new Comparator<HRegionInfo>() {
374 public int compare(HRegionInfo r1, HRegionInfo r2) {
375 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
376 }
377 });
378
379 int numRegions = admin.getTableRegions(TABLE_NAME).size();
380 int numRegionsAfterMerge = numRegions - 2;
381 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
382 regions.get(2).getEncodedNameAsBytes(), true);
383 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
384 regions.get(6).getEncodedNameAsBytes(), true);
385
386 waitRegionsAfterMerge(numRegionsAfterMerge);
387 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
388
389
390 String snapshotName = "snapshotAfterMerge";
391 SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME.getNameAsString(),
392 SnapshotDescription.Type.FLUSH, 3);
393
394
395 TableName cloneName = TableName.valueOf("cloneMerge");
396 admin.cloneSnapshot(snapshotName, cloneName);
397 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName);
398
399 MobSnapshotTestingUtils.verifyMobRowCount(UTIL, TABLE_NAME, numRows);
400 MobSnapshotTestingUtils.verifyMobRowCount(UTIL, cloneName, numRows);
401
402
403 UTIL.deleteTable(cloneName);
404 }
405
406
407
408
409 @Test (timeout=300000)
410 public void testFlushCreateListDestroy() throws Exception {
411 LOG.debug("------- Starting Snapshot test -------------");
412 Admin admin = UTIL.getHBaseAdmin();
413
414 SnapshotTestingUtils.assertNoSnapshots(admin);
415
416 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
417
418 String snapshotName = "flushSnapshotCreateListDestroy";
419 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
420 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
421 SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM),
422 snapshotName, rootDir, fs, true);
423 }
424
425
426
427
428
429
430 @Test(timeout=300000)
431 public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
432 final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
433 final TableName TABLE2_NAME =
434 TableName.valueOf(STRING_TABLE2_NAME);
435
436 int ssNum = 20;
437 Admin admin = UTIL.getHBaseAdmin();
438
439 SnapshotTestingUtils.assertNoSnapshots(admin);
440
441 SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
442
443 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
444 SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
445
446 final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
447
448 class SSRunnable implements Runnable {
449 SnapshotDescription ss;
450 SSRunnable(SnapshotDescription ss) {
451 this.ss = ss;
452 }
453
454 @Override
455 public void run() {
456 try {
457 Admin admin = UTIL.getHBaseAdmin();
458 LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
459 admin.takeSnapshotAsync(ss);
460 } catch (Exception e) {
461 LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
462 ss)
463 + ". This is ok, we expect some", e);
464 }
465 LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
466 toBeSubmitted.countDown();
467 }
468 };
469
470
471 SnapshotDescription[] descs = new SnapshotDescription[ssNum];
472 for (int i = 0; i < ssNum; i++) {
473 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
474 builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
475 builder.setName("ss"+i);
476 builder.setType(SnapshotDescription.Type.FLUSH);
477 descs[i] = builder.build();
478 }
479
480
481 for (int i=0 ; i < ssNum; i++) {
482 new Thread(new SSRunnable(descs[i])).start();
483 }
484
485
486 toBeSubmitted.await();
487
488
489 while (true) {
490 int doneCount = 0;
491 for (SnapshotDescription ss : descs) {
492 try {
493 if (admin.isSnapshotFinished(ss)) {
494 doneCount++;
495 }
496 } catch (Exception e) {
497 LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
498 doneCount++;
499 }
500 }
501 if (doneCount == descs.length) {
502 break;
503 }
504 Thread.sleep(100);
505 }
506
507
508 logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
509
510 List<SnapshotDescription> taken = admin.listSnapshots();
511 int takenSize = taken.size();
512 LOG.info("Taken " + takenSize + " snapshots: " + taken);
513 assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
514 " issued many requests", takenSize < ssNum && takenSize > 0);
515
516
517 int t1SnapshotsCount = 0;
518 int t2SnapshotsCount = 0;
519 for (SnapshotDescription ss : taken) {
520 if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
521 t1SnapshotsCount++;
522 } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
523 t2SnapshotsCount++;
524 }
525 }
526 assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
527 assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
528
529 UTIL.deleteTable(TABLE2_NAME);
530 }
531
532 private void logFSTree(Path root) throws IOException {
533 FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
534 }
535
536 private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
537 throws IOException, InterruptedException {
538 Admin admin = UTIL.getHBaseAdmin();
539
540 long startTime = System.currentTimeMillis();
541 while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
542
543
544 if ((System.currentTimeMillis() - startTime) > 15000)
545 break;
546 Thread.sleep(100);
547 }
548 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
549 }
550 }
551