1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
22 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
23 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
24 import static org.junit.Assert.assertEquals;
25 import static org.junit.Assert.assertFalse;
26 import static org.junit.Assert.assertNotEquals;
27 import static org.junit.Assert.assertNotNull;
28 import static org.junit.Assert.assertTrue;
29 import static org.junit.Assert.fail;
30
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.Collection;
35 import java.util.HashMap;
36 import java.util.HashSet;
37 import java.util.LinkedList;
38 import java.util.List;
39 import java.util.Map;
40 import java.util.Random;
41 import java.util.Map.Entry;
42 import java.util.NavigableMap;
43 import java.util.Set;
44 import java.util.UUID;
45 import java.util.concurrent.Callable;
46 import java.util.concurrent.CountDownLatch;
47 import java.util.concurrent.ExecutorService;
48 import java.util.concurrent.Executors;
49 import java.util.concurrent.Future;
50 import java.util.concurrent.ScheduledThreadPoolExecutor;
51 import java.util.concurrent.SynchronousQueue;
52 import java.util.concurrent.ThreadPoolExecutor;
53 import java.util.concurrent.TimeUnit;
54 import java.util.concurrent.atomic.AtomicBoolean;
55
56 import org.apache.commons.io.IOUtils;
57 import org.apache.commons.logging.Log;
58 import org.apache.commons.logging.LogFactory;
59 import org.apache.hadoop.conf.Configuration;
60 import org.apache.hadoop.fs.FileStatus;
61 import org.apache.hadoop.fs.FileSystem;
62 import org.apache.hadoop.fs.Path;
63 import org.apache.hadoop.hbase.ClusterStatus;
64 import org.apache.hadoop.hbase.HBaseTestingUtility;
65 import org.apache.hadoop.hbase.HColumnDescriptor;
66 import org.apache.hadoop.hbase.HConstants;
67 import org.apache.hadoop.hbase.HRegionInfo;
68 import org.apache.hadoop.hbase.HRegionLocation;
69 import org.apache.hadoop.hbase.HTableDescriptor;
70 import org.apache.hadoop.hbase.TableExistsException;
71 import org.apache.hadoop.hbase.testclassification.LargeTests;
72 import org.apache.hadoop.hbase.MiniHBaseCluster;
73 import org.apache.hadoop.hbase.RegionLocations;
74 import org.apache.hadoop.hbase.ServerName;
75 import org.apache.hadoop.hbase.TableName;
76 import org.apache.hadoop.hbase.MetaTableAccessor;
77 import org.apache.hadoop.hbase.client.Admin;
78 import org.apache.hadoop.hbase.client.ClusterConnection;
79 import org.apache.hadoop.hbase.client.Connection;
80 import org.apache.hadoop.hbase.client.ConnectionFactory;
81 import org.apache.hadoop.hbase.client.Delete;
82 import org.apache.hadoop.hbase.client.Durability;
83 import org.apache.hadoop.hbase.client.Get;
84 import org.apache.hadoop.hbase.client.HBaseAdmin;
85 import org.apache.hadoop.hbase.client.HConnection;
86 import org.apache.hadoop.hbase.client.HTable;
87 import org.apache.hadoop.hbase.client.MetaScanner;
88 import org.apache.hadoop.hbase.client.Mutation;
89 import org.apache.hadoop.hbase.client.Put;
90 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
91 import org.apache.hadoop.hbase.client.Result;
92 import org.apache.hadoop.hbase.client.ResultScanner;
93 import org.apache.hadoop.hbase.client.RowMutations;
94 import org.apache.hadoop.hbase.client.Scan;
95 import org.apache.hadoop.hbase.client.Table;
96 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
97 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
98 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
99 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
100 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
101 import org.apache.hadoop.hbase.io.hfile.TestHFile;
102 import org.apache.hadoop.hbase.master.AssignmentManager;
103 import org.apache.hadoop.hbase.master.HMaster;
104 import org.apache.hadoop.hbase.master.RegionState;
105 import org.apache.hadoop.hbase.master.RegionStates;
106 import org.apache.hadoop.hbase.master.TableLockManager;
107 import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
108 import org.apache.hadoop.hbase.mob.MobFileName;
109 import org.apache.hadoop.hbase.mob.MobUtils;
110 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
111 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
112 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
113 import org.apache.hadoop.hbase.regionserver.HRegion;
114 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
115 import org.apache.hadoop.hbase.regionserver.HRegionServer;
116 import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl;
117 import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction;
118 import org.apache.hadoop.hbase.replication.ReplicationFactory;
119 import org.apache.hadoop.hbase.replication.ReplicationQueues;
120 import org.apache.hadoop.hbase.testclassification.LargeTests;
121 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
122 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
123 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
124 import org.apache.hadoop.hbase.util.HBaseFsck.PrintingErrorReporter;
125 import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
126 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
127 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
128 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
129 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
130 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
131 import org.apache.zookeeper.KeeperException;
132 import org.junit.AfterClass;
133 import org.junit.Assert;
134 import org.junit.Before;
135 import org.junit.BeforeClass;
136 import org.junit.Ignore;
137 import org.junit.Test;
138 import org.junit.experimental.categories.Category;
139 import org.junit.rules.TestName;
140
141 import com.google.common.collect.Multimap;
142
143
144
145
146 @Category(LargeTests.class)
147 public class TestHBaseFsck {
148 static final int POOL_SIZE = 7;
149 private static final Log LOG = LogFactory.getLog(TestHBaseFsck.class);
150 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
151 private final static Configuration conf = TEST_UTIL.getConfiguration();
152 private final static String FAM_STR = "fam";
153 private final static byte[] FAM = Bytes.toBytes(FAM_STR);
154 private final static int REGION_ONLINE_TIMEOUT = 800;
155 private static RegionStates regionStates;
156 private static ExecutorService tableExecutorService;
157 private static ScheduledThreadPoolExecutor hbfsckExecutorService;
158 private static ClusterConnection connection;
159 private static Admin admin;
160
161
162 private HTable tbl;
163 private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"),
164 Bytes.toBytes("B"), Bytes.toBytes("C") };
165
166 private final static byte[][] ROWKEYS= new byte[][] {
167 Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
168 Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
169
170 @BeforeClass
171 public static void setUpBeforeClass() throws Exception {
172 TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
173 MasterSyncObserver.class.getName());
174
175 conf.setInt("hbase.regionserver.handler.count", 2);
176 conf.setInt("hbase.regionserver.metahandler.count", 30);
177
178 conf.setInt("hbase.htable.threads.max", POOL_SIZE);
179 conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
180 conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
181 conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
182 conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
183 conf.setInt("hfile.format.version", 3);
184 TEST_UTIL.startMiniCluster(3);
185
186 tableExecutorService = new ThreadPoolExecutor(1, POOL_SIZE, 60, TimeUnit.SECONDS,
187 new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("testhbck"));
188
189 hbfsckExecutorService = new ScheduledThreadPoolExecutor(POOL_SIZE);
190
191 AssignmentManager assignmentManager =
192 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
193 regionStates = assignmentManager.getRegionStates();
194
195 connection = (ClusterConnection) TEST_UTIL.getConnection();
196
197 admin = connection.getAdmin();
198 admin.setBalancerRunning(false, true);
199
200 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME);
201 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.NAMESPACE_TABLE_NAME);
202 }
203
204 @AfterClass
205 public static void tearDownAfterClass() throws Exception {
206 tableExecutorService.shutdown();
207 hbfsckExecutorService.shutdown();
208 admin.close();
209 TEST_UTIL.shutdownMiniCluster();
210 }
211
212 @Before
213 public void setUp() {
214 EnvironmentEdgeManager.reset();
215 }
216
217 @Test (timeout=180000)
218 public void testHBaseFsck() throws Exception {
219 assertNoErrors(doFsck(conf, false));
220 TableName table = TableName.valueOf("tableBadMetaAssign");
221 HTableDescriptor desc = new HTableDescriptor(table);
222 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
223 desc.addFamily(hcd);
224 createTable(TEST_UTIL, desc, null);
225
226
227 assertNoErrors(doFsck(conf, false));
228
229
230
231 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
232 Scan scan = new Scan();
233 scan.setStartRow(Bytes.toBytes(table+",,"));
234 ResultScanner scanner = meta.getScanner(scan);
235 HRegionInfo hri = null;
236
237 Result res = scanner.next();
238 ServerName currServer =
239 ServerName.parseFrom(res.getValue(HConstants.CATALOG_FAMILY,
240 HConstants.SERVER_QUALIFIER));
241 long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
242 HConstants.STARTCODE_QUALIFIER));
243
244 for (JVMClusterUtil.RegionServerThread rs :
245 TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {
246
247 ServerName sn = rs.getRegionServer().getServerName();
248
249
250 if (!currServer.getHostAndPort().equals(sn.getHostAndPort()) ||
251 startCode != sn.getStartcode()) {
252 Put put = new Put(res.getRow());
253 put.setDurability(Durability.SKIP_WAL);
254 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
255 Bytes.toBytes(sn.getHostAndPort()));
256 put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
257 Bytes.toBytes(sn.getStartcode()));
258 meta.put(put);
259 hri = MetaTableAccessor.getHRegionInfo(res);
260 break;
261 }
262 }
263
264
265 assertErrors(doFsck(conf, true), new ERROR_CODE[]{
266 ERROR_CODE.SERVER_DOES_NOT_MATCH_META});
267
268 TEST_UTIL.getHBaseCluster().getMaster()
269 .getAssignmentManager().waitForAssignment(hri);
270
271
272 assertNoErrors(doFsck(conf, false));
273
274
275 Table t = connection.getTable(table, tableExecutorService);
276 ResultScanner s = t.getScanner(new Scan());
277 s.close();
278 t.close();
279
280 scanner.close();
281 meta.close();
282 }
283
284 @Test(timeout=180000)
285 public void testFixAssignmentsWhenMETAinTransition() throws Exception {
286 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
287 admin.closeRegion(cluster.getServerHoldingMeta(), HRegionInfo.FIRST_META_REGIONINFO);
288 regionStates.regionOffline(HRegionInfo.FIRST_META_REGIONINFO);
289 new MetaTableLocator().deleteMetaLocation(cluster.getMaster().getZooKeeper());
290 assertFalse(regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO));
291 HBaseFsck hbck = doFsck(conf, true);
292 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNKNOWN, ERROR_CODE.NO_META_REGION,
293 ERROR_CODE.NULL_META_REGION });
294 assertNoErrors(doFsck(conf, false));
295 }
296
297
298
299
300 private HRegionInfo createRegion(final HTableDescriptor
301 htd, byte[] startKey, byte[] endKey)
302 throws IOException {
303 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
304 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
305 MetaTableAccessor.addRegionToMeta(meta, hri);
306 meta.close();
307 return hri;
308 }
309
310
311
312
313 private void dumpMeta(TableName tableName) throws IOException {
314 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName);
315 for (byte[] row : metaRows) {
316 LOG.info(Bytes.toString(row));
317 }
318 }
319
320
321
322
323
324 private void undeployRegion(Connection conn, ServerName sn,
325 HRegionInfo hri) throws IOException, InterruptedException {
326 try {
327 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) conn, sn, hri);
328 if (!hri.isMetaTable()) {
329 admin.offline(hri.getRegionName());
330 }
331 } catch (IOException ioe) {
332 LOG.warn("Got exception when attempting to offline region "
333 + Bytes.toString(hri.getRegionName()), ioe);
334 }
335 }
336
337
338
339
340
341
342 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
343 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
344 boolean hdfs) throws IOException, InterruptedException {
345 deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false, HRegionInfo.DEFAULT_REPLICA_ID);
346 }
347
348
349
350
351
352
353
354
355
356 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
357 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
358 boolean hdfs, boolean regionInfoOnly, int replicaId)
359 throws IOException, InterruptedException {
360 LOG.info("** Before delete:");
361 dumpMeta(htd.getTableName());
362
363 List<HRegionLocation> locations = tbl.getAllRegionLocations();
364 for (HRegionLocation location : locations) {
365 HRegionInfo hri = location.getRegionInfo();
366 ServerName hsa = location.getServerName();
367 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
368 && Bytes.compareTo(hri.getEndKey(), endKey) == 0
369 && hri.getReplicaId() == replicaId) {
370
371 LOG.info("RegionName: " +hri.getRegionNameAsString());
372 byte[] deleteRow = hri.getRegionName();
373
374 if (unassign) {
375 LOG.info("Undeploying region " + hri + " from server " + hsa);
376 undeployRegion(connection, hsa, hri);
377 }
378
379 if (regionInfoOnly) {
380 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
381 Path rootDir = FSUtils.getRootDir(conf);
382 FileSystem fs = rootDir.getFileSystem(conf);
383 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
384 hri.getEncodedName());
385 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
386 fs.delete(hriPath, true);
387 }
388
389 if (hdfs) {
390 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
391 Path rootDir = FSUtils.getRootDir(conf);
392 FileSystem fs = rootDir.getFileSystem(conf);
393 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
394 hri.getEncodedName());
395 HBaseFsck.debugLsr(conf, p);
396 boolean success = fs.delete(p, true);
397 LOG.info("Deleted " + p + " sucessfully? " + success);
398 HBaseFsck.debugLsr(conf, p);
399 }
400
401 if (metaRow) {
402 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
403 Delete delete = new Delete(deleteRow);
404 meta.delete(delete);
405 }
406 }
407 }
408 LOG.info(hri.toString() + hsa.toString());
409 }
410
411 TEST_UTIL.getMetaTableRows(htd.getTableName());
412 LOG.info("*** After delete:");
413 dumpMeta(htd.getTableName());
414 }
415
416
417
418
419
420
421
422
423
424
425 void setupTable(TableName tablename) throws Exception {
426 setupTableWithRegionReplica(tablename, 1);
427 }
428
429
430
431
432
433
434
435
436
437
438 void setupTableWithRegionReplica(TableName tablename, int replicaCount) throws Exception {
439 HTableDescriptor desc = new HTableDescriptor(tablename);
440 desc.setRegionReplication(replicaCount);
441 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
442 desc.addFamily(hcd);
443 createTable(TEST_UTIL, desc, SPLITS);
444
445 tbl = (HTable) connection.getTable(tablename, tableExecutorService);
446 List<Put> puts = new ArrayList<Put>();
447 for (byte[] row : ROWKEYS) {
448 Put p = new Put(row);
449 p.add(FAM, Bytes.toBytes("val"), row);
450 puts.add(p);
451 }
452 tbl.put(puts);
453 tbl.flushCommits();
454 }
455
456
457
458
459
460
461
462 void setupMobTable(TableName tablename) throws Exception {
463 HTableDescriptor desc = new HTableDescriptor(tablename);
464 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
465 hcd.setMobEnabled(true);
466 hcd.setMobThreshold(0);
467 desc.addFamily(hcd);
468 admin.createTable(desc, SPLITS);
469
470 tbl = (HTable) connection.getTable(tablename, tableExecutorService);
471 List<Put> puts = new ArrayList<Put>();
472 for (byte[] row : ROWKEYS) {
473 Put p = new Put(row);
474 p.add(FAM, Bytes.toBytes("val"), row);
475 puts.add(p);
476 }
477 tbl.put(puts);
478 tbl.flushCommits();
479 }
480
481
482
483
484 int countRows() throws IOException {
485 Scan s = new Scan();
486 ResultScanner rs = tbl.getScanner(s);
487 int i = 0;
488 while(rs.next() !=null) {
489 i++;
490 }
491 return i;
492 }
493
494
495
496
497
498
499
500 void cleanupTable(TableName tablename) throws Exception {
501 if (tbl != null) {
502 tbl.close();
503 tbl = null;
504 }
505
506 ((ClusterConnection) connection).clearRegionCache();
507 deleteTable(TEST_UTIL, tablename);
508 }
509
510
511
512
513 @Test (timeout=180000)
514 public void testHBaseFsckClean() throws Exception {
515 assertNoErrors(doFsck(conf, false));
516 TableName table = TableName.valueOf("tableClean");
517 try {
518 HBaseFsck hbck = doFsck(conf, false);
519 assertNoErrors(hbck);
520
521 setupTable(table);
522 assertEquals(ROWKEYS.length, countRows());
523
524
525 hbck = doFsck(conf, false);
526 assertNoErrors(hbck);
527 assertEquals(0, hbck.getOverlapGroups(table).size());
528 assertEquals(ROWKEYS.length, countRows());
529 } finally {
530 cleanupTable(table);
531 }
532 }
533
534
535
536
537 @Test (timeout=180000)
538 public void testHbckThreadpooling() throws Exception {
539 TableName table =
540 TableName.valueOf("tableDupeStartKey");
541 try {
542
543 setupTable(table);
544
545
546 Configuration newconf = new Configuration(conf);
547 newconf.setInt("hbasefsck.numthreads", 1);
548 assertNoErrors(doFsck(newconf, false));
549
550
551 } finally {
552 cleanupTable(table);
553 }
554 }
555
556 @Test (timeout=180000)
557 public void testHbckFixOrphanTable() throws Exception {
558 TableName table = TableName.valueOf("tableInfo");
559 FileSystem fs = null;
560 Path tableinfo = null;
561 try {
562 setupTable(table);
563
564 Path hbaseTableDir = FSUtils.getTableDir(
565 FSUtils.getRootDir(conf), table);
566 fs = hbaseTableDir.getFileSystem(conf);
567 FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
568 tableinfo = status.getPath();
569 fs.rename(tableinfo, new Path("/.tableinfo"));
570
571
572 HBaseFsck hbck = doFsck(conf, false);
573 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });
574
575
576 hbck = doFsck(conf, true);
577 assertNoErrors(hbck);
578 status = null;
579 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
580 assertNotNull(status);
581
582 HTableDescriptor htd = admin.getTableDescriptor(table);
583 htd.setValue("NOT_DEFAULT", "true");
584 admin.disableTable(table);
585 admin.modifyTable(table, htd);
586 admin.enableTable(table);
587 fs.delete(status.getPath(), true);
588
589
590 htd = admin.getTableDescriptor(table);
591 hbck = doFsck(conf, true);
592 assertNoErrors(hbck);
593 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
594 assertNotNull(status);
595 htd = admin.getTableDescriptor(table);
596 assertEquals(htd.getValue("NOT_DEFAULT"), "true");
597 } finally {
598 fs.rename(new Path("/.tableinfo"), tableinfo);
599 cleanupTable(table);
600 }
601 }
602
603
604
605
606
607
608 @Test (timeout=180000)
609 public void testParallelHbck() throws Exception {
610 final ExecutorService service;
611 final Future<HBaseFsck> hbck1,hbck2;
612
613 class RunHbck implements Callable<HBaseFsck>{
614 boolean fail = true;
615 @Override
616 public HBaseFsck call(){
617 Configuration c = new Configuration(conf);
618 c.setInt("hbase.hbck.lockfile.attempts", 1);
619
620
621 c.setInt("hbase.hbck.lockfile.maxwaittime", 3);
622 try{
623 return doFsck(c, true);
624 } catch(Exception e){
625 if (e.getMessage().contains("Duplicate hbck")) {
626 fail = false;
627 }
628 }
629
630 if (fail) fail();
631 return null;
632 }
633 }
634 service = Executors.newFixedThreadPool(2);
635 hbck1 = service.submit(new RunHbck());
636 hbck2 = service.submit(new RunHbck());
637 service.shutdown();
638
639 service.awaitTermination(15, TimeUnit.SECONDS);
640 HBaseFsck h1 = hbck1.get();
641 HBaseFsck h2 = hbck2.get();
642
643 assert(h1 == null || h2 == null);
644 if (h1 != null) {
645 assert(h1.getRetCode() >= 0);
646 }
647 if (h2 != null) {
648 assert(h2.getRetCode() >= 0);
649 }
650 }
651
652
653
654
655
656
657
658 @Test (timeout=180000)
659 public void testParallelWithRetriesHbck() throws Exception {
660 final ExecutorService service;
661 final Future<HBaseFsck> hbck1,hbck2;
662
663
664
665
666
667
668
669 final int timeoutInSeconds = 80;
670 final int sleepIntervalInMilliseconds = 200;
671 final int maxSleepTimeInMilliseconds = 6000;
672 final int maxRetryAttempts = 15;
673
674 class RunHbck implements Callable<HBaseFsck>{
675
676 @Override
677 public HBaseFsck call() throws Exception {
678
679 Configuration c = new Configuration(conf);
680 c.setInt("hbase.hbck.lockfile.maxwaittime", timeoutInSeconds);
681 c.setInt("hbase.hbck.lockfile.attempt.sleep.interval", sleepIntervalInMilliseconds);
682 c.setInt("hbase.hbck.lockfile.attempt.maxsleeptime", maxSleepTimeInMilliseconds);
683 c.setInt("hbase.hbck.lockfile.attempts", maxRetryAttempts);
684 return doFsck(c, false);
685 }
686 }
687
688 service = Executors.newFixedThreadPool(2);
689 hbck1 = service.submit(new RunHbck());
690 hbck2 = service.submit(new RunHbck());
691 service.shutdown();
692
693 service.awaitTermination(timeoutInSeconds * 2, TimeUnit.SECONDS);
694 HBaseFsck h1 = hbck1.get();
695 HBaseFsck h2 = hbck2.get();
696
697 assertNotNull(h1);
698 assertNotNull(h2);
699 assert(h1.getRetCode() >= 0);
700 assert(h2.getRetCode() >= 0);
701
702 }
703
704
705
706
707
708 @Test (timeout=180000)
709 public void testDupeStartKey() throws Exception {
710 TableName table =
711 TableName.valueOf("tableDupeStartKey");
712 try {
713 setupTable(table);
714 assertNoErrors(doFsck(conf, false));
715 assertEquals(ROWKEYS.length, countRows());
716
717
718 HRegionInfo hriDupe =
719 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("A2"));
720 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
721 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
722 .waitForAssignment(hriDupe);
723 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
724 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
725
726 HBaseFsck hbck = doFsck(conf, false);
727 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
728 ERROR_CODE.DUPE_STARTKEYS});
729 assertEquals(2, hbck.getOverlapGroups(table).size());
730 assertEquals(ROWKEYS.length, countRows());
731
732
733 doFsck(conf,true);
734
735
736 HBaseFsck hbck2 = doFsck(conf,false);
737 assertNoErrors(hbck2);
738 assertEquals(0, hbck2.getOverlapGroups(table).size());
739 assertEquals(ROWKEYS.length, countRows());
740 } finally {
741 cleanupTable(table);
742 }
743 }
744
745
746
747
748
749 @Test (timeout=180000)
750 public void testHbckWithRegionReplica() throws Exception {
751 TableName table =
752 TableName.valueOf("testHbckWithRegionReplica");
753 try {
754 setupTableWithRegionReplica(table, 2);
755 TEST_UTIL.getHBaseAdmin().flush(table.getName());
756 assertNoErrors(doFsck(conf, false));
757 } finally {
758 cleanupTable(table);
759 }
760 }
761
762 @Test
763 public void testHbckWithFewerReplica() throws Exception {
764 TableName table =
765 TableName.valueOf("testHbckWithFewerReplica");
766 try {
767 setupTableWithRegionReplica(table, 2);
768 TEST_UTIL.getHBaseAdmin().flush(table.getName());
769 assertNoErrors(doFsck(conf, false));
770 assertEquals(ROWKEYS.length, countRows());
771 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
772 Bytes.toBytes("C"), true, false, false, false, 1);
773
774 HBaseFsck hbck = doFsck(conf, false);
775 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.NOT_DEPLOYED});
776
777 hbck = doFsck(conf, true);
778
779 hbck = doFsck(conf, false);
780 assertErrors(hbck, new ERROR_CODE[]{});
781 } finally {
782 cleanupTable(table);
783 }
784 }
785
786 @Test
787 public void testHbckWithExcessReplica() throws Exception {
788 TableName table =
789 TableName.valueOf("testHbckWithExcessReplica");
790 try {
791 setupTableWithRegionReplica(table, 2);
792 TEST_UTIL.getHBaseAdmin().flush(table.getName());
793 assertNoErrors(doFsck(conf, false));
794 assertEquals(ROWKEYS.length, countRows());
795
796
797
798
799 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
800 List<HRegionInfo> regions = TEST_UTIL.getHBaseAdmin().getTableRegions(table);
801 byte[] startKey = Bytes.toBytes("B");
802 byte[] endKey = Bytes.toBytes("C");
803 byte[] metaKey = null;
804 HRegionInfo newHri = null;
805 for (HRegionInfo h : regions) {
806 if (Bytes.compareTo(h.getStartKey(), startKey) == 0 &&
807 Bytes.compareTo(h.getEndKey(), endKey) == 0 &&
808 h.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
809 metaKey = h.getRegionName();
810
811 newHri = RegionReplicaUtil.getRegionInfoForReplica(h, 2);
812 break;
813 }
814 }
815 Put put = new Put(metaKey);
816 ServerName sn = TEST_UTIL.getHBaseAdmin().getClusterStatus().getServers()
817 .toArray(new ServerName[0])[0];
818
819 MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), -1, 2);
820 meta.put(put);
821 meta.flushCommits();
822
823 HBaseFsckRepair.fixUnassigned((HBaseAdmin)TEST_UTIL.getHBaseAdmin(), newHri);
824 HBaseFsckRepair.waitUntilAssigned((HBaseAdmin)TEST_UTIL.getHBaseAdmin(), newHri);
825
826 Delete delete = new Delete(metaKey);
827 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(2));
828 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(2));
829 delete.deleteColumns(HConstants.CATALOG_FAMILY, MetaTableAccessor.getSeqNumColumn(2));
830 meta.delete(delete);
831 meta.flushCommits();
832 meta.close();
833
834 HBaseFsck hbck = doFsck(conf, false);
835 assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.NOT_IN_META});
836
837 hbck = doFsck(conf, true);
838
839 hbck = doFsck(conf, false);
840 assertErrors(hbck, new ERROR_CODE[]{});
841 } finally {
842 cleanupTable(table);
843 }
844 }
845
846
847
848 Map<ServerName, List<String>> getDeployedHRIs(final HBaseAdmin admin) throws IOException {
849 ClusterStatus status = admin.getClusterStatus();
850 Collection<ServerName> regionServers = status.getServers();
851 Map<ServerName, List<String>> mm =
852 new HashMap<ServerName, List<String>>();
853 for (ServerName hsi : regionServers) {
854 AdminProtos.AdminService.BlockingInterface server = ((HConnection) connection).getAdmin(hsi);
855
856
857 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
858 List<String> regionNames = new ArrayList<String>();
859 for (HRegionInfo hri : regions) {
860 regionNames.add(hri.getRegionNameAsString());
861 }
862 mm.put(hsi, regionNames);
863 }
864 return mm;
865 }
866
867
868
869
870 ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) {
871 for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) {
872 if (e.getValue().contains(hri.getRegionNameAsString())) {
873 return e.getKey();
874 }
875 }
876 return null;
877 }
878
879
880
881
882
883 @Test (timeout=180000)
884 public void testDupeRegion() throws Exception {
885 TableName table =
886 TableName.valueOf("tableDupeRegion");
887 try {
888 setupTable(table);
889 assertNoErrors(doFsck(conf, false));
890 assertEquals(ROWKEYS.length, countRows());
891
892
893 HRegionInfo hriDupe =
894 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("B"));
895
896 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
897 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
898 .waitForAssignment(hriDupe);
899 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
900 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
901
902
903
904
905
906 while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriDupe) == null) {
907 Thread.sleep(250);
908 }
909
910 LOG.debug("Finished assignment of dupe region");
911
912
913 HBaseFsck hbck = doFsck(conf, false);
914 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
915 ERROR_CODE.DUPE_STARTKEYS});
916 assertEquals(2, hbck.getOverlapGroups(table).size());
917 assertEquals(ROWKEYS.length, countRows());
918
919
920 doFsck(conf,true);
921
922
923 HBaseFsck hbck2 = doFsck(conf,false);
924 assertNoErrors(hbck2);
925 assertEquals(0, hbck2.getOverlapGroups(table).size());
926 assertEquals(ROWKEYS.length, countRows());
927 } finally {
928 cleanupTable(table);
929 }
930 }
931
932
933
934
935 @Test (timeout=180000)
936 public void testDegenerateRegions() throws Exception {
937 TableName table = TableName.valueOf("tableDegenerateRegions");
938 try {
939 setupTable(table);
940 assertNoErrors(doFsck(conf,false));
941 assertEquals(ROWKEYS.length, countRows());
942
943
944 HRegionInfo hriDupe =
945 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("B"));
946 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
947 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
948 .waitForAssignment(hriDupe);
949 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
950 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
951
952 HBaseFsck hbck = doFsck(conf,false);
953 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION, ERROR_CODE.DUPE_STARTKEYS,
954 ERROR_CODE.DUPE_STARTKEYS });
955 assertEquals(2, hbck.getOverlapGroups(table).size());
956 assertEquals(ROWKEYS.length, countRows());
957
958
959 doFsck(conf,true);
960
961
962 HBaseFsck hbck2 = doFsck(conf,false);
963 assertNoErrors(hbck2);
964 assertEquals(0, hbck2.getOverlapGroups(table).size());
965 assertEquals(ROWKEYS.length, countRows());
966 } finally {
967 cleanupTable(table);
968 }
969 }
970
971
972
973
974
975 @Test (timeout=180000)
976 public void testContainedRegionOverlap() throws Exception {
977 TableName table =
978 TableName.valueOf("tableContainedRegionOverlap");
979 try {
980 setupTable(table);
981 assertEquals(ROWKEYS.length, countRows());
982
983
984 HRegionInfo hriOverlap =
985 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
986 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
987 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
988 .waitForAssignment(hriOverlap);
989 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
990 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
991
992 HBaseFsck hbck = doFsck(conf, false);
993 assertErrors(hbck, new ERROR_CODE[] {
994 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
995 assertEquals(2, hbck.getOverlapGroups(table).size());
996 assertEquals(ROWKEYS.length, countRows());
997
998
999 doFsck(conf, true);
1000
1001
1002 HBaseFsck hbck2 = doFsck(conf,false);
1003 assertNoErrors(hbck2);
1004 assertEquals(0, hbck2.getOverlapGroups(table).size());
1005 assertEquals(ROWKEYS.length, countRows());
1006 } finally {
1007 cleanupTable(table);
1008 }
1009 }
1010
1011
1012
1013
1014
1015
1016
1017 @Test (timeout=180000)
1018 public void testSidelineOverlapRegion() throws Exception {
1019 TableName table =
1020 TableName.valueOf("testSidelineOverlapRegion");
1021 try {
1022 setupTable(table);
1023 assertEquals(ROWKEYS.length, countRows());
1024
1025
1026 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1027 HMaster master = cluster.getMaster();
1028 HRegionInfo hriOverlap1 =
1029 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("AB"));
1030 master.assignRegion(hriOverlap1);
1031 master.getAssignmentManager().waitForAssignment(hriOverlap1);
1032 HRegionInfo hriOverlap2 =
1033 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("AB"), Bytes.toBytes("B"));
1034 master.assignRegion(hriOverlap2);
1035 master.getAssignmentManager().waitForAssignment(hriOverlap2);
1036
1037 HBaseFsck hbck = doFsck(conf, false);
1038 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.DUPE_STARTKEYS,
1039 ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
1040 assertEquals(3, hbck.getOverlapGroups(table).size());
1041 assertEquals(ROWKEYS.length, countRows());
1042
1043
1044 Multimap<byte[], HbckInfo> overlapGroups = hbck.getOverlapGroups(table);
1045 ServerName serverName = null;
1046 byte[] regionName = null;
1047 for (HbckInfo hbi: overlapGroups.values()) {
1048 if ("A".equals(Bytes.toString(hbi.getStartKey()))
1049 && "B".equals(Bytes.toString(hbi.getEndKey()))) {
1050 regionName = hbi.getRegionName();
1051
1052
1053 int k = cluster.getServerWith(regionName);
1054 for (int i = 0; i < 3; i++) {
1055 if (i != k) {
1056 HRegionServer rs = cluster.getRegionServer(i);
1057 serverName = rs.getServerName();
1058 break;
1059 }
1060 }
1061
1062 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) connection,
1063 cluster.getRegionServer(k).getServerName(), hbi.getHdfsHRI());
1064 admin.offline(regionName);
1065 break;
1066 }
1067 }
1068
1069 assertNotNull(regionName);
1070 assertNotNull(serverName);
1071 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) {
1072 Put put = new Put(regionName);
1073 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
1074 Bytes.toBytes(serverName.getHostAndPort()));
1075 meta.put(put);
1076 }
1077
1078
1079 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
1080 fsck.connect();
1081 fsck.setDisplayFullReport();
1082 fsck.setTimeLag(0);
1083 fsck.setFixAssignments(true);
1084 fsck.setFixMeta(true);
1085 fsck.setFixHdfsHoles(true);
1086 fsck.setFixHdfsOverlaps(true);
1087 fsck.setFixHdfsOrphans(true);
1088 fsck.setFixVersionFile(true);
1089 fsck.setSidelineBigOverlaps(true);
1090 fsck.setMaxMerge(2);
1091 fsck.onlineHbck();
1092 fsck.close();
1093
1094
1095
1096 HBaseFsck hbck2 = doFsck(conf,false);
1097 assertNoErrors(hbck2);
1098 assertEquals(0, hbck2.getOverlapGroups(table).size());
1099 assertTrue(ROWKEYS.length > countRows());
1100 } finally {
1101 cleanupTable(table);
1102 }
1103 }
1104
1105
1106
1107
1108
1109 @Test (timeout=180000)
1110 public void testOverlapAndOrphan() throws Exception {
1111 TableName table =
1112 TableName.valueOf("tableOverlapAndOrphan");
1113 try {
1114 setupTable(table);
1115 assertEquals(ROWKEYS.length, countRows());
1116
1117
1118 admin.disableTable(table);
1119 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1120 Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1121 TEST_UTIL.getHBaseAdmin().enableTable(table);
1122
1123 HRegionInfo hriOverlap =
1124 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
1125 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1126 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1127 .waitForAssignment(hriOverlap);
1128 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1129 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1130
1131 HBaseFsck hbck = doFsck(conf, false);
1132 assertErrors(hbck, new ERROR_CODE[] {
1133 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1134 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1135
1136
1137 doFsck(conf, true);
1138
1139
1140 HBaseFsck hbck2 = doFsck(conf,false);
1141 assertNoErrors(hbck2);
1142 assertEquals(0, hbck2.getOverlapGroups(table).size());
1143 assertEquals(ROWKEYS.length, countRows());
1144 } finally {
1145 cleanupTable(table);
1146 }
1147 }
1148
1149
1150
1151
1152
1153
1154 @Test (timeout=180000)
1155 public void testCoveredStartKey() throws Exception {
1156 TableName table =
1157 TableName.valueOf("tableCoveredStartKey");
1158 try {
1159 setupTable(table);
1160 assertEquals(ROWKEYS.length, countRows());
1161
1162
1163 HRegionInfo hriOverlap =
1164 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B2"));
1165 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1166 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1167 .waitForAssignment(hriOverlap);
1168 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1169 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1170
1171 HBaseFsck hbck = doFsck(conf, false);
1172 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
1173 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
1174 assertEquals(3, hbck.getOverlapGroups(table).size());
1175 assertEquals(ROWKEYS.length, countRows());
1176
1177
1178 doFsck(conf, true);
1179
1180
1181 HBaseFsck hbck2 = doFsck(conf, false);
1182 assertErrors(hbck2, new ERROR_CODE[0]);
1183 assertEquals(0, hbck2.getOverlapGroups(table).size());
1184 assertEquals(ROWKEYS.length, countRows());
1185 } finally {
1186 cleanupTable(table);
1187 }
1188 }
1189
1190
1191
1192
1193
1194 @Test (timeout=180000)
1195 public void testRegionHole() throws Exception {
1196 TableName table =
1197 TableName.valueOf("tableRegionHole");
1198 try {
1199 setupTable(table);
1200 assertEquals(ROWKEYS.length, countRows());
1201
1202
1203 admin.disableTable(table);
1204 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1205 Bytes.toBytes("C"), true, true, true);
1206 admin.enableTable(table);
1207
1208 HBaseFsck hbck = doFsck(conf, false);
1209 assertErrors(hbck, new ERROR_CODE[] {
1210 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1211
1212 assertEquals(0, hbck.getOverlapGroups(table).size());
1213
1214
1215 doFsck(conf, true);
1216
1217
1218 assertNoErrors(doFsck(conf,false));
1219 assertEquals(ROWKEYS.length - 2 , countRows());
1220 } finally {
1221 cleanupTable(table);
1222 }
1223 }
1224
1225
1226
1227
1228
1229 @Test (timeout=180000)
1230 public void testHDFSRegioninfoMissing() throws Exception {
1231 TableName table = TableName.valueOf("tableHDFSRegioninfoMissing");
1232 try {
1233 setupTable(table);
1234 assertEquals(ROWKEYS.length, countRows());
1235
1236
1237 admin.disableTable(table);
1238 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1239 Bytes.toBytes("C"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
1240 TEST_UTIL.getHBaseAdmin().enableTable(table);
1241
1242 HBaseFsck hbck = doFsck(conf, false);
1243 assertErrors(hbck, new ERROR_CODE[] {
1244 ERROR_CODE.ORPHAN_HDFS_REGION,
1245 ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1246 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1247
1248 assertEquals(0, hbck.getOverlapGroups(table).size());
1249
1250
1251 doFsck(conf, true);
1252
1253
1254 assertNoErrors(doFsck(conf, false));
1255 assertEquals(ROWKEYS.length, countRows());
1256 } finally {
1257 cleanupTable(table);
1258 }
1259 }
1260
1261
1262
1263
1264
1265 @Test (timeout=180000)
1266 public void testNotInMetaOrDeployedHole() throws Exception {
1267 TableName table =
1268 TableName.valueOf("tableNotInMetaOrDeployedHole");
1269 try {
1270 setupTable(table);
1271 assertEquals(ROWKEYS.length, countRows());
1272
1273
1274 admin.disableTable(table);
1275 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1276 Bytes.toBytes("C"), true, true, false);
1277 admin.enableTable(table);
1278
1279 HBaseFsck hbck = doFsck(conf, false);
1280 assertErrors(hbck, new ERROR_CODE[] {
1281 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1282
1283 assertEquals(0, hbck.getOverlapGroups(table).size());
1284
1285
1286 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1287 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1288
1289
1290 assertNoErrors(doFsck(conf,false));
1291 assertEquals(ROWKEYS.length, countRows());
1292 } finally {
1293 cleanupTable(table);
1294 }
1295 }
1296
1297
1298
1299
1300 @Test (timeout=180000)
1301 public void testNotInMetaHole() throws Exception {
1302 TableName table =
1303 TableName.valueOf("tableNotInMetaHole");
1304 try {
1305 setupTable(table);
1306 assertEquals(ROWKEYS.length, countRows());
1307
1308
1309 admin.disableTable(table);
1310 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1311 Bytes.toBytes("C"), false, true, false);
1312 admin.enableTable(table);
1313
1314 HBaseFsck hbck = doFsck(conf, false);
1315 assertErrors(hbck, new ERROR_CODE[] {
1316 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1317
1318 assertEquals(0, hbck.getOverlapGroups(table).size());
1319
1320
1321 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1322 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1323
1324
1325 assertNoErrors(doFsck(conf,false));
1326 assertEquals(ROWKEYS.length, countRows());
1327 } finally {
1328 cleanupTable(table);
1329 }
1330 }
1331
1332
1333
1334
1335
1336 @Test (timeout=180000)
1337 public void testNotInHdfs() throws Exception {
1338 TableName table =
1339 TableName.valueOf("tableNotInHdfs");
1340 try {
1341 setupTable(table);
1342 assertEquals(ROWKEYS.length, countRows());
1343
1344
1345 admin.flush(table);
1346
1347
1348 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1349 Bytes.toBytes("C"), false, false, true);
1350
1351 HBaseFsck hbck = doFsck(conf, false);
1352 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1353
1354 assertEquals(0, hbck.getOverlapGroups(table).size());
1355
1356
1357 doFsck(conf, true);
1358
1359
1360 assertNoErrors(doFsck(conf,false));
1361 assertEquals(ROWKEYS.length - 2, countRows());
1362 } finally {
1363 cleanupTable(table);
1364 }
1365 }
1366
1367
1368
1369
1370
1371 @Test (timeout=180000)
1372 public void testNotInHdfsWithReplicas() throws Exception {
1373 TableName table =
1374 TableName.valueOf("tableNotInHdfs");
1375 HBaseAdmin admin = new HBaseAdmin(conf);
1376 try {
1377 HRegionInfo[] oldHris = new HRegionInfo[2];
1378 setupTableWithRegionReplica(table, 2);
1379 assertEquals(ROWKEYS.length, countRows());
1380 NavigableMap<HRegionInfo, ServerName> map = MetaScanner.allTableRegions(TEST_UTIL.getConnection(),
1381 tbl.getName());
1382 int i = 0;
1383
1384 for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
1385 if (m.getKey().getStartKey().length > 0 &&
1386 m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
1387 LOG.debug("Initially server hosting " + m.getKey() + " is " + m.getValue());
1388 oldHris[i++] = m.getKey();
1389 }
1390 }
1391
1392 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1393
1394
1395 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1396 Bytes.toBytes("C"), false, false, true);
1397
1398 HBaseFsck hbck = doFsck(conf, false);
1399 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1400
1401
1402 doFsck(conf, true);
1403
1404
1405 assertNoErrors(doFsck(conf,false));
1406 assertEquals(ROWKEYS.length - 2, countRows());
1407
1408
1409
1410 i = 0;
1411 HRegionInfo[] newHris = new HRegionInfo[2];
1412
1413 map = MetaScanner.allTableRegions(TEST_UTIL.getConnection(), tbl.getName());
1414
1415 for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
1416 if (m.getKey().getStartKey().length > 0 &&
1417 m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
1418 newHris[i++] = m.getKey();
1419 }
1420 }
1421
1422 Collection<ServerName> servers = admin.getClusterStatus().getServers();
1423 Set<HRegionInfo> onlineRegions = new HashSet<HRegionInfo>();
1424 for (ServerName s : servers) {
1425 List<HRegionInfo> list = admin.getOnlineRegions(s);
1426 onlineRegions.addAll(list);
1427 }
1428
1429 assertTrue(onlineRegions.containsAll(Arrays.asList(newHris)));
1430
1431
1432 assertFalse(onlineRegions.removeAll(Arrays.asList(oldHris)));
1433 } finally {
1434 cleanupTable(table);
1435 admin.close();
1436 }
1437 }
1438
1439
1440
1441
1442
1443
1444 @Test (timeout=180000)
1445 public void testNoHdfsTable() throws Exception {
1446 TableName table = TableName.valueOf("NoHdfsTable");
1447 setupTable(table);
1448 assertEquals(ROWKEYS.length, countRows());
1449
1450
1451 admin.flush(table);
1452
1453
1454 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""),
1455 Bytes.toBytes("A"), false, false, true);
1456 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1457 Bytes.toBytes("B"), false, false, true);
1458 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1459 Bytes.toBytes("C"), false, false, true);
1460 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"),
1461 Bytes.toBytes(""), false, false, true);
1462
1463
1464 deleteTableDir(table);
1465
1466 HBaseFsck hbck = doFsck(conf, false);
1467 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS,
1468 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS,
1469 ERROR_CODE.NOT_IN_HDFS,});
1470
1471 assertEquals(0, hbck.getOverlapGroups(table).size());
1472
1473
1474 doFsck(conf, true);
1475
1476
1477 assertNoErrors(doFsck(conf,false));
1478 assertFalse("Table " + table + " should have been deleted", admin.tableExists(table));
1479 }
1480
1481 public void deleteTableDir(TableName table) throws IOException {
1482 Path rootDir = FSUtils.getRootDir(conf);
1483 FileSystem fs = rootDir.getFileSystem(conf);
1484 Path p = FSUtils.getTableDir(rootDir, table);
1485 HBaseFsck.debugLsr(conf, p);
1486 boolean success = fs.delete(p, true);
1487 LOG.info("Deleted " + p + " sucessfully? " + success);
1488 }
1489
1490
1491
1492
1493 @Test (timeout=180000)
1494 public void testNoVersionFile() throws Exception {
1495
1496 Path rootDir = FSUtils.getRootDir(conf);
1497 FileSystem fs = rootDir.getFileSystem(conf);
1498 Path versionFile = new Path(rootDir, HConstants.VERSION_FILE_NAME);
1499 fs.delete(versionFile, true);
1500
1501
1502 HBaseFsck hbck = doFsck(conf, false);
1503 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_VERSION_FILE });
1504
1505 doFsck(conf, true);
1506
1507
1508 assertNoErrors(doFsck(conf, false));
1509 }
1510
1511
1512
1513
1514 @Test (timeout=180000)
1515 public void testRegionShouldNotBeDeployed() throws Exception {
1516 TableName table =
1517 TableName.valueOf("tableRegionShouldNotBeDeployed");
1518 try {
1519 LOG.info("Starting testRegionShouldNotBeDeployed.");
1520 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1521 assertTrue(cluster.waitForActiveAndReadyMaster());
1522
1523
1524 byte[][] SPLIT_KEYS = new byte[][] { new byte[0], Bytes.toBytes("aaa"),
1525 Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), Bytes.toBytes("ddd") };
1526 HTableDescriptor htdDisabled = new HTableDescriptor(table);
1527 htdDisabled.addFamily(new HColumnDescriptor(FAM));
1528
1529
1530 FSTableDescriptors fstd = new FSTableDescriptors(conf);
1531 fstd.createTableDescriptor(htdDisabled);
1532 List<HRegionInfo> disabledRegions =
1533 TEST_UTIL.createMultiRegionsInMeta(conf, htdDisabled, SPLIT_KEYS);
1534
1535
1536 HRegionServer hrs = cluster.getRegionServer(0);
1537
1538
1539 admin.disableTable(table);
1540 admin.enableTable(table);
1541
1542
1543 admin.disableTable(table);
1544 HRegionInfo region = disabledRegions.remove(0);
1545 byte[] regionName = region.getRegionName();
1546
1547
1548 assertTrue(cluster.getServerWith(regionName) == -1);
1549
1550
1551
1552
1553
1554 HRegion r = HRegion.openHRegion(
1555 region, htdDisabled, hrs.getWAL(region), conf);
1556 hrs.addToOnlineRegions(r);
1557
1558 HBaseFsck hbck = doFsck(conf, false);
1559 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.SHOULD_NOT_BE_DEPLOYED });
1560
1561
1562 doFsck(conf, true);
1563
1564
1565 assertNoErrors(doFsck(conf, false));
1566 } finally {
1567 admin.enableTable(table);
1568 cleanupTable(table);
1569 }
1570 }
1571
1572
1573
1574
1575 @Test (timeout=180000)
1576 public void testFixByTable() throws Exception {
1577 TableName table1 =
1578 TableName.valueOf("testFixByTable1");
1579 TableName table2 =
1580 TableName.valueOf("testFixByTable2");
1581 try {
1582 setupTable(table1);
1583
1584 admin.flush(table1);
1585
1586 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1587 Bytes.toBytes("C"), false, false, true);
1588
1589 setupTable(table2);
1590
1591 admin.flush(table2);
1592
1593 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1594 Bytes.toBytes("C"), false, false, true);
1595
1596 HBaseFsck hbck = doFsck(conf, false);
1597 assertErrors(hbck, new ERROR_CODE[] {
1598 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS});
1599
1600
1601 doFsck(conf, true, table1);
1602
1603 assertNoErrors(doFsck(conf, false, table1));
1604
1605 assertErrors(doFsck(conf, false, table2),
1606 new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1607
1608
1609 doFsck(conf, true, table2);
1610
1611 assertNoErrors(doFsck(conf, false));
1612 assertEquals(ROWKEYS.length - 2, countRows());
1613 } finally {
1614 cleanupTable(table1);
1615 cleanupTable(table2);
1616 }
1617 }
1618
1619
1620
1621 @Test (timeout=180000)
1622 public void testLingeringSplitParent() throws Exception {
1623 TableName table =
1624 TableName.valueOf("testLingeringSplitParent");
1625 Table meta = null;
1626 try {
1627 setupTable(table);
1628 assertEquals(ROWKEYS.length, countRows());
1629
1630
1631 admin.flush(table);
1632 HRegionLocation location = tbl.getRegionLocation("B");
1633
1634
1635 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1636 Bytes.toBytes("C"), true, true, false);
1637
1638
1639 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1640 HRegionInfo hri = location.getRegionInfo();
1641
1642 HRegionInfo a = new HRegionInfo(tbl.getName(),
1643 Bytes.toBytes("B"), Bytes.toBytes("BM"));
1644 HRegionInfo b = new HRegionInfo(tbl.getName(),
1645 Bytes.toBytes("BM"), Bytes.toBytes("C"));
1646
1647 hri.setOffline(true);
1648 hri.setSplit(true);
1649
1650 MetaTableAccessor.addRegionToMeta(meta, hri, a, b);
1651 meta.close();
1652 admin.flush(TableName.META_TABLE_NAME);
1653
1654 HBaseFsck hbck = doFsck(conf, false);
1655 assertErrors(hbck, new ERROR_CODE[] {
1656 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1657
1658
1659 hbck = doFsck(conf, true);
1660 assertErrors(hbck, new ERROR_CODE[] {
1661 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN });
1662 assertFalse(hbck.shouldRerun());
1663 hbck = doFsck(conf, false);
1664 assertErrors(hbck, new ERROR_CODE[] {
1665 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1666
1667
1668 hbck = new HBaseFsck(conf, hbfsckExecutorService);
1669 hbck.connect();
1670 hbck.setDisplayFullReport();
1671 hbck.setTimeLag(0);
1672 hbck.setFixSplitParents(true);
1673 hbck.onlineHbck();
1674 assertTrue(hbck.shouldRerun());
1675 hbck.close();
1676
1677 Get get = new Get(hri.getRegionName());
1678 Result result = meta.get(get);
1679 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1680 HConstants.SPLITA_QUALIFIER).isEmpty());
1681 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1682 HConstants.SPLITB_QUALIFIER).isEmpty());
1683 admin.flush(TableName.META_TABLE_NAME);
1684
1685
1686 doFsck(conf, true);
1687
1688
1689 assertNoErrors(doFsck(conf, false));
1690 assertEquals(ROWKEYS.length, countRows());
1691 } finally {
1692 cleanupTable(table);
1693 IOUtils.closeQuietly(meta);
1694 }
1695 }
1696
1697
1698
1699
1700
1701 @Test (timeout=180000)
1702 public void testValidLingeringSplitParent() throws Exception {
1703 TableName table =
1704 TableName.valueOf("testLingeringSplitParent");
1705 Table meta = null;
1706 try {
1707 setupTable(table);
1708 assertEquals(ROWKEYS.length, countRows());
1709
1710
1711 admin.flush(table);
1712 HRegionLocation location = tbl.getRegionLocation(Bytes.toBytes("B"));
1713
1714 meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1715 HRegionInfo hri = location.getRegionInfo();
1716
1717
1718 byte[] regionName = location.getRegionInfo().getRegionName();
1719 admin.splitRegion(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
1720 TestEndToEndSplitTransaction.blockUntilRegionSplit(conf, 60000, regionName, true);
1721
1722
1723
1724
1725 HBaseFsck hbck = doFsck(
1726 conf, true, true, false, false, false, true, true, true, false, false, false, false, null);
1727 assertErrors(hbck, new ERROR_CODE[] {});
1728
1729
1730 Get get = new Get(hri.getRegionName());
1731 Result result = meta.get(get);
1732 assertNotNull(result);
1733 assertNotNull(MetaTableAccessor.getHRegionInfo(result));
1734
1735 assertEquals(ROWKEYS.length, countRows());
1736
1737
1738 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
1739 assertNoErrors(doFsck(conf, false));
1740 } finally {
1741 cleanupTable(table);
1742 IOUtils.closeQuietly(meta);
1743 }
1744 }
1745
1746
1747
1748
1749
1750 @Test(timeout=75000)
1751 public void testSplitDaughtersNotInMeta() throws Exception {
1752 TableName table = TableName.valueOf("testSplitdaughtersNotInMeta");
1753 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
1754 try {
1755 setupTable(table);
1756 assertEquals(ROWKEYS.length, countRows());
1757
1758
1759 admin.flush(table);
1760 HRegionLocation location = tbl.getRegionLocation(Bytes.toBytes("B"));
1761
1762 HRegionInfo hri = location.getRegionInfo();
1763
1764
1765
1766 admin.enableCatalogJanitor(false);
1767
1768
1769 byte[] regionName = location.getRegionInfo().getRegionName();
1770 admin.splitRegion(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
1771 TestEndToEndSplitTransaction.blockUntilRegionSplit(conf, 60000, regionName, true);
1772
1773 PairOfSameType<HRegionInfo> daughters =
1774 MetaTableAccessor.getDaughterRegions(meta.get(new Get(regionName)));
1775
1776
1777 Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
1778 undeployRegion(connection, hris.get(daughters.getFirst()), daughters.getFirst());
1779 undeployRegion(connection, hris.get(daughters.getSecond()), daughters.getSecond());
1780
1781 List<Delete> deletes = new ArrayList<>();
1782 deletes.add(new Delete(daughters.getFirst().getRegionName()));
1783 deletes.add(new Delete(daughters.getSecond().getRegionName()));
1784 meta.delete(deletes);
1785
1786
1787 RegionStates regionStates = TEST_UTIL.getMiniHBaseCluster().getMaster().
1788 getAssignmentManager().getRegionStates();
1789 regionStates.deleteRegion(daughters.getFirst());
1790 regionStates.deleteRegion(daughters.getSecond());
1791
1792 HBaseFsck hbck = doFsck(conf, false);
1793 assertErrors(hbck,
1794 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1795 ERROR_CODE.HOLE_IN_REGION_CHAIN });
1796
1797
1798 hbck = doFsck(
1799 conf, true, true, false, false, false, false, false, false, false, false, false,false,null);
1800 assertErrors(hbck,
1801 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1802 ERROR_CODE.HOLE_IN_REGION_CHAIN });
1803
1804
1805 Get get = new Get(hri.getRegionName());
1806 Result result = meta.get(get);
1807 assertNotNull(result);
1808 assertNotNull(MetaTableAccessor.getHRegionInfo(result));
1809
1810 assertEquals(ROWKEYS.length, countRows());
1811
1812
1813 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
1814 assertNoErrors(doFsck(conf, false));
1815 } finally {
1816 admin.enableCatalogJanitor(true);
1817 meta.close();
1818 cleanupTable(table);
1819 }
1820 }
1821
1822
1823
1824
1825
1826 @Test(timeout=120000)
1827 public void testMissingFirstRegion() throws Exception {
1828 TableName table = TableName.valueOf("testMissingFirstRegion");
1829 try {
1830 setupTable(table);
1831 assertEquals(ROWKEYS.length, countRows());
1832
1833
1834 admin.disableTable(table);
1835 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes("A"), true,
1836 true, true);
1837 admin.enableTable(table);
1838
1839 HBaseFsck hbck = doFsck(conf, false);
1840 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY });
1841
1842 doFsck(conf, true);
1843
1844 assertNoErrors(doFsck(conf, false));
1845 } finally {
1846 cleanupTable(table);
1847 }
1848 }
1849
1850
1851
1852
1853
1854 @Test(timeout=120000)
1855 public void testRegionDeployedNotInHdfs() throws Exception {
1856 TableName table =
1857 TableName.valueOf("testSingleRegionDeployedNotInHdfs");
1858 try {
1859 setupTable(table);
1860 admin.flush(table);
1861
1862
1863 deleteRegion(conf, tbl.getTableDescriptor(),
1864 HConstants.EMPTY_START_ROW, Bytes.toBytes("A"), false,
1865 false, true);
1866
1867 HBaseFsck hbck = doFsck(conf, false);
1868 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
1869
1870 doFsck(conf, true);
1871
1872 assertNoErrors(doFsck(conf, false));
1873 } finally {
1874 cleanupTable(table);
1875 }
1876 }
1877
1878
1879
1880
1881
1882 @Test(timeout=120000)
1883 public void testMissingLastRegion() throws Exception {
1884 TableName table =
1885 TableName.valueOf("testMissingLastRegion");
1886 try {
1887 setupTable(table);
1888 assertEquals(ROWKEYS.length, countRows());
1889
1890
1891 admin.disableTable(table);
1892 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""), true,
1893 true, true);
1894 admin.enableTable(table);
1895
1896 HBaseFsck hbck = doFsck(conf, false);
1897 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY });
1898
1899 doFsck(conf, true);
1900
1901 assertNoErrors(doFsck(conf, false));
1902 } finally {
1903 cleanupTable(table);
1904 }
1905 }
1906
1907
1908
1909
1910 @Test (timeout=180000)
1911 public void testFixAssignmentsAndNoHdfsChecking() throws Exception {
1912 TableName table =
1913 TableName.valueOf("testFixAssignmentsAndNoHdfsChecking");
1914 try {
1915 setupTable(table);
1916 assertEquals(ROWKEYS.length, countRows());
1917
1918
1919 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1920 Bytes.toBytes("B"), true, false, false, false, HRegionInfo.DEFAULT_REPLICA_ID);
1921
1922
1923 HBaseFsck hbck = doFsck(conf, false);
1924 assertErrors(hbck, new ERROR_CODE[] {
1925 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1926
1927
1928 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
1929 fsck.connect();
1930 fsck.setDisplayFullReport();
1931 fsck.setTimeLag(0);
1932 fsck.setCheckHdfs(false);
1933 fsck.onlineHbck();
1934 assertErrors(fsck, new ERROR_CODE[] {
1935 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1936 fsck.close();
1937
1938
1939 fsck = new HBaseFsck(conf, hbfsckExecutorService);
1940 fsck.connect();
1941 fsck.setDisplayFullReport();
1942 fsck.setTimeLag(0);
1943 fsck.setCheckHdfs(false);
1944 fsck.setFixAssignments(true);
1945 fsck.onlineHbck();
1946 assertTrue(fsck.shouldRerun());
1947 fsck.onlineHbck();
1948 assertNoErrors(fsck);
1949
1950 assertEquals(ROWKEYS.length, countRows());
1951
1952 fsck.close();
1953 } finally {
1954 cleanupTable(table);
1955 }
1956 }
1957
1958
1959
1960
1961
1962
1963 @Test (timeout=180000)
1964 public void testFixMetaNotWorkingWithNoHdfsChecking() throws Exception {
1965 TableName table =
1966 TableName.valueOf("testFixMetaNotWorkingWithNoHdfsChecking");
1967 try {
1968 setupTable(table);
1969 assertEquals(ROWKEYS.length, countRows());
1970
1971
1972 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1973 Bytes.toBytes("B"), false, true, false, false, HRegionInfo.DEFAULT_REPLICA_ID);
1974
1975
1976 HBaseFsck hbck = doFsck(conf, false);
1977 assertErrors(hbck,
1978 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
1979
1980
1981 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
1982 fsck.connect();
1983 fsck.setDisplayFullReport();
1984 fsck.setTimeLag(0);
1985 fsck.setCheckHdfs(false);
1986 fsck.onlineHbck();
1987 assertErrors(fsck,
1988 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
1989 fsck.close();
1990
1991
1992 fsck = new HBaseFsck(conf, hbfsckExecutorService);
1993 fsck.connect();
1994 fsck.setDisplayFullReport();
1995 fsck.setTimeLag(0);
1996 fsck.setCheckHdfs(false);
1997 fsck.setFixAssignments(true);
1998 fsck.setFixMeta(true);
1999 fsck.onlineHbck();
2000 assertFalse(fsck.shouldRerun());
2001 assertErrors(fsck,
2002 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN });
2003 fsck.close();
2004
2005
2006 fsck = doFsck(conf, true);
2007 assertTrue(fsck.shouldRerun());
2008 fsck = doFsck(conf, true);
2009 assertNoErrors(fsck);
2010 } finally {
2011 cleanupTable(table);
2012 }
2013 }
2014
2015
2016
2017
2018
2019 @Test (timeout=180000)
2020 public void testFixHdfsHolesNotWorkingWithNoHdfsChecking() throws Exception {
2021 TableName table =
2022 TableName.valueOf("testFixHdfsHolesNotWorkingWithNoHdfsChecking");
2023 try {
2024 setupTable(table);
2025 assertEquals(ROWKEYS.length, countRows());
2026
2027
2028 admin.disableTable(table);
2029 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
2030 Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID);
2031 TEST_UTIL.getHBaseAdmin().enableTable(table);
2032
2033 HRegionInfo hriOverlap =
2034 createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B"));
2035 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
2036 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
2037 .waitForAssignment(hriOverlap);
2038 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
2039 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
2040
2041 HBaseFsck hbck = doFsck(conf, false);
2042 assertErrors(hbck, new ERROR_CODE[] {
2043 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2044 ERROR_CODE.HOLE_IN_REGION_CHAIN});
2045
2046
2047 HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
2048 fsck.connect();
2049 fsck.setDisplayFullReport();
2050 fsck.setTimeLag(0);
2051 fsck.setCheckHdfs(false);
2052 fsck.onlineHbck();
2053 assertErrors(fsck, new ERROR_CODE[] {
2054 ERROR_CODE.HOLE_IN_REGION_CHAIN});
2055 fsck.close();
2056
2057
2058 fsck = new HBaseFsck(conf, hbfsckExecutorService);
2059 fsck.connect();
2060 fsck.setDisplayFullReport();
2061 fsck.setTimeLag(0);
2062 fsck.setCheckHdfs(false);
2063 fsck.setFixHdfsHoles(true);
2064 fsck.setFixHdfsOverlaps(true);
2065 fsck.setFixHdfsOrphans(true);
2066 fsck.onlineHbck();
2067 assertFalse(fsck.shouldRerun());
2068 assertErrors(fsck, new ERROR_CODE[] { ERROR_CODE.HOLE_IN_REGION_CHAIN});
2069 fsck.close();
2070 } finally {
2071 if (admin.isTableDisabled(table)) {
2072 admin.enableTable(table);
2073 }
2074 cleanupTable(table);
2075 }
2076 }
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086 Path getFlushedHFile(FileSystem fs, TableName table) throws IOException {
2087 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2088 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2089 Path famDir = new Path(regionDir, FAM_STR);
2090
2091
2092 while (true) {
2093 FileStatus[] hfFss = fs.listStatus(famDir);
2094 if (hfFss.length == 0) {
2095 continue;
2096 }
2097 for (FileStatus hfs : hfFss) {
2098 if (!hfs.isDirectory()) {
2099 return hfs.getPath();
2100 }
2101 }
2102 }
2103 }
2104
2105
2106
2107
2108
2109
2110
2111
2112 Path getFlushedMobFile(FileSystem fs, TableName table) throws IOException {
2113 Path regionDir = MobUtils.getMobRegionPath(conf, table);
2114 Path famDir = new Path(regionDir, FAM_STR);
2115
2116
2117 while (true) {
2118 FileStatus[] hfFss = fs.listStatus(famDir);
2119 if (hfFss.length == 0) {
2120 continue;
2121 }
2122 for (FileStatus hfs : hfFss) {
2123 if (!hfs.isDirectory()) {
2124 return hfs.getPath();
2125 }
2126 }
2127 }
2128 }
2129
2130
2131
2132
2133
2134
2135 String createMobFileName(String oldFileName) {
2136 MobFileName mobFileName = MobFileName.create(oldFileName);
2137 String startKey = mobFileName.getStartKey();
2138 String date = mobFileName.getDate();
2139 return MobFileName.create(startKey, date, UUID.randomUUID().toString().replaceAll("-", ""))
2140 .getFileName();
2141 }
2142
2143
2144
2145
2146 @Test(timeout=180000)
2147 public void testQuarantineCorruptHFile() throws Exception {
2148 TableName table = TableName.valueOf(name.getMethodName());
2149 try {
2150 setupTable(table);
2151 assertEquals(ROWKEYS.length, countRows());
2152 admin.flush(table);
2153
2154 FileSystem fs = FileSystem.get(conf);
2155 Path hfile = getFlushedHFile(fs, table);
2156
2157
2158 admin.disableTable(table);
2159
2160
2161 Path corrupt = new Path(hfile.getParent(), "deadbeef");
2162 TestHFile.truncateFile(fs, hfile, corrupt);
2163 LOG.info("Created corrupted file " + corrupt);
2164 HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));
2165
2166
2167 HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
2168 assertEquals(res.getRetCode(), 0);
2169 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2170 assertEquals(hfcc.getHFilesChecked(), 5);
2171 assertEquals(hfcc.getCorrupted().size(), 1);
2172 assertEquals(hfcc.getFailures().size(), 0);
2173 assertEquals(hfcc.getQuarantined().size(), 1);
2174 assertEquals(hfcc.getMissing().size(), 0);
2175
2176
2177 admin.enableTable(table);
2178 } finally {
2179 cleanupTable(table);
2180 }
2181 }
2182
2183
2184
2185
2186 @Test(timeout=180000)
2187 public void testQuarantineCorruptMobFile() throws Exception {
2188 TableName table = TableName.valueOf(name.getMethodName());
2189 try {
2190 setupMobTable(table);
2191 assertEquals(ROWKEYS.length, countRows());
2192 admin.flush(table);
2193
2194 FileSystem fs = FileSystem.get(conf);
2195 Path mobFile = getFlushedMobFile(fs, table);
2196 admin.disableTable(table);
2197
2198 String corruptMobFile = createMobFileName(mobFile.getName());
2199 Path corrupt = new Path(mobFile.getParent(), corruptMobFile);
2200 TestHFile.truncateFile(fs, mobFile, corrupt);
2201 LOG.info("Created corrupted mob file " + corrupt);
2202 HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));
2203 HBaseFsck.debugLsr(conf, MobUtils.getMobHome(conf));
2204
2205
2206 admin.enableTable(table);
2207 HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
2208 assertEquals(res.getRetCode(), 0);
2209 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2210 assertEquals(hfcc.getHFilesChecked(), 4);
2211 assertEquals(hfcc.getCorrupted().size(), 0);
2212 assertEquals(hfcc.getFailures().size(), 0);
2213 assertEquals(hfcc.getQuarantined().size(), 0);
2214 assertEquals(hfcc.getMissing().size(), 0);
2215 assertEquals(hfcc.getMobFilesChecked(), 5);
2216 assertEquals(hfcc.getCorruptedMobFiles().size(), 1);
2217 assertEquals(hfcc.getFailureMobFiles().size(), 0);
2218 assertEquals(hfcc.getQuarantinedMobFiles().size(), 1);
2219 assertEquals(hfcc.getMissedMobFiles().size(), 0);
2220 String quarantinedMobFile = hfcc.getQuarantinedMobFiles().iterator().next().getName();
2221 assertEquals(corruptMobFile, quarantinedMobFile);
2222 } finally {
2223 cleanupTable(table);
2224 }
2225 }
2226
2227
2228
2229
2230 private void doQuarantineTest(TableName table, HBaseFsck hbck, int check,
2231 int corrupt, int fail, int quar, int missing) throws Exception {
2232 try {
2233 setupTable(table);
2234 assertEquals(ROWKEYS.length, countRows());
2235 admin.flush(table);
2236
2237
2238 admin.disableTable(table);
2239
2240 String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission",
2241 table.getNameAsString()};
2242 HBaseFsck res = hbck.exec(hbfsckExecutorService, args);
2243
2244 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
2245 assertEquals(hfcc.getHFilesChecked(), check);
2246 assertEquals(hfcc.getCorrupted().size(), corrupt);
2247 assertEquals(hfcc.getFailures().size(), fail);
2248 assertEquals(hfcc.getQuarantined().size(), quar);
2249 assertEquals(hfcc.getMissing().size(), missing);
2250
2251
2252 admin.enableTableAsync(table);
2253 while (!admin.isTableEnabled(table)) {
2254 try {
2255 Thread.sleep(250);
2256 } catch (InterruptedException e) {
2257 e.printStackTrace();
2258 fail("Interrupted when trying to enable table " + table);
2259 }
2260 }
2261 } finally {
2262 cleanupTable(table);
2263 }
2264 }
2265
2266
2267
2268
2269
2270 @Test(timeout=180000)
2271 public void testQuarantineMissingHFile() throws Exception {
2272 TableName table = TableName.valueOf(name.getMethodName());
2273
2274
2275 final FileSystem fs = FileSystem.get(conf);
2276 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2277 @Override
2278 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
2279 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2280 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2281 @Override
2282 protected void checkHFile(Path p) throws IOException {
2283 if (attemptedFirstHFile.compareAndSet(false, true)) {
2284 assertTrue(fs.delete(p, true));
2285 }
2286 super.checkHFile(p);
2287 }
2288 };
2289 }
2290 };
2291 doQuarantineTest(table, hbck, 4, 0, 0, 0, 1);
2292 hbck.close();
2293 }
2294
2295
2296
2297
2298
2299
2300
2301 @Ignore @Test(timeout=180000)
2302 public void testQuarantineMissingFamdir() throws Exception {
2303 TableName table = TableName.valueOf(name.getMethodName());
2304
2305 final FileSystem fs = FileSystem.get(conf);
2306 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2307 @Override
2308 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
2309 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2310 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2311 @Override
2312 protected void checkColFamDir(Path p) throws IOException {
2313 if (attemptedFirstHFile.compareAndSet(false, true)) {
2314 assertTrue(fs.delete(p, true));
2315 }
2316 super.checkColFamDir(p);
2317 }
2318 };
2319 }
2320 };
2321 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
2322 hbck.close();
2323 }
2324
2325 @Test(timeout=60000)
2326 public void testCheckReplication() throws Exception {
2327
2328 HBaseFsck hbck = doFsck(conf, false);
2329 assertNoErrors(hbck);
2330
2331
2332 ReplicationAdmin replicationAdmin = new ReplicationAdmin(conf);
2333 Assert.assertEquals(0, replicationAdmin.getPeersCount());
2334 String zkPort = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
2335 replicationAdmin.addPeer("1", "127.0.0.1:2181" + zkPort + ":/hbase");
2336 replicationAdmin.getPeersCount();
2337 Assert.assertEquals(1, replicationAdmin.getPeersCount());
2338
2339
2340 ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "Test Hbase Fsck", connection);
2341 ReplicationQueues repQueues =
2342 ReplicationFactory.getReplicationQueues(zkw, conf, connection);
2343 repQueues.init("server1");
2344
2345 repQueues.addLog("1", "file1");
2346 repQueues.addLog("1-server2", "file1");
2347 Assert.assertEquals(2, repQueues.getAllQueues().size());
2348 hbck = doFsck(conf, false);
2349 assertNoErrors(hbck);
2350
2351
2352 repQueues.addLog("2", "file1");
2353 repQueues.addLog("2-server2", "file1");
2354 Assert.assertEquals(4, repQueues.getAllQueues().size());
2355 hbck = doFsck(conf, false);
2356 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNDELETED_REPLICATION_QUEUE,
2357 ERROR_CODE.UNDELETED_REPLICATION_QUEUE });
2358
2359
2360 hbck = doFsck(conf, true);
2361 hbck = doFsck(conf, false);
2362 assertNoErrors(hbck);
2363
2364 Assert.assertEquals(2, repQueues.getAllQueues().size());
2365 Assert.assertNull(repQueues.getLogsInQueue("2"));
2366 Assert.assertNull(repQueues.getLogsInQueue("2-sever2"));
2367
2368 replicationAdmin.removePeer("1");
2369 repQueues.removeAllQueues();
2370 zkw.close();
2371 replicationAdmin.close();
2372 }
2373
2374
2375
2376
2377
2378 @Test(timeout=180000)
2379 public void testQuarantineMissingRegionDir() throws Exception {
2380 TableName table = TableName.valueOf(name.getMethodName());
2381
2382 final FileSystem fs = FileSystem.get(conf);
2383 HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
2384 @Override
2385 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles)
2386 throws IOException {
2387 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
2388 AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
2389 @Override
2390 protected void checkRegionDir(Path p) throws IOException {
2391 if (attemptedFirstHFile.compareAndSet(false, true)) {
2392 assertTrue(fs.delete(p, true));
2393 }
2394 super.checkRegionDir(p);
2395 }
2396 };
2397 }
2398 };
2399 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
2400 hbck.close();
2401 }
2402
2403
2404
2405
2406 @Test (timeout=180000)
2407 public void testLingeringReferenceFile() throws Exception {
2408 TableName table =
2409 TableName.valueOf("testLingeringReferenceFile");
2410 try {
2411 setupTable(table);
2412 assertEquals(ROWKEYS.length, countRows());
2413
2414
2415 FileSystem fs = FileSystem.get(conf);
2416 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
2417 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
2418 Path famDir = new Path(regionDir, FAM_STR);
2419 Path fakeReferenceFile = new Path(famDir, "fbce357483ceea.12144538");
2420 fs.create(fakeReferenceFile);
2421
2422 HBaseFsck hbck = doFsck(conf, false);
2423 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LINGERING_REFERENCE_HFILE });
2424
2425 doFsck(conf, true);
2426
2427 assertNoErrors(doFsck(conf, false));
2428 } finally {
2429 cleanupTable(table);
2430 }
2431 }
2432
2433
2434
2435
2436 @Test (timeout=180000)
2437 public void testMissingRegionInfoQualifier() throws Exception {
2438 Connection connection = ConnectionFactory.createConnection(conf);
2439 TableName table = TableName.valueOf("testMissingRegionInfoQualifier");
2440 try {
2441 setupTable(table);
2442
2443
2444 final List<Delete> deletes = new LinkedList<Delete>();
2445 Table meta = connection.getTable(TableName.META_TABLE_NAME, hbfsckExecutorService);
2446 MetaScanner.metaScan(connection, new MetaScanner.MetaScannerVisitor() {
2447
2448 @Override
2449 public boolean processRow(Result rowResult) throws IOException {
2450 HRegionInfo hri = MetaTableAccessor.getHRegionInfo(rowResult);
2451 if (hri != null && !hri.getTable().isSystemTable()) {
2452 Delete delete = new Delete(rowResult.getRow());
2453 delete.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2454 deletes.add(delete);
2455 }
2456 return true;
2457 }
2458
2459 @Override
2460 public void close() throws IOException {
2461 }
2462 });
2463 meta.delete(deletes);
2464
2465
2466 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
2467 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes("node1:60020")));
2468 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
2469 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(1362150791183L)));
2470 meta.close();
2471
2472 HBaseFsck hbck = doFsck(conf, false);
2473 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
2474
2475
2476 hbck = doFsck(conf, true);
2477
2478
2479 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
2480 } finally {
2481 cleanupTable(table);
2482 }
2483 connection.close();
2484 }
2485
2486
2487
2488
2489
2490 @Test (timeout=180000)
2491 public void testErrorReporter() throws Exception {
2492 try {
2493 MockErrorReporter.calledCount = 0;
2494 doFsck(conf, false);
2495 assertEquals(MockErrorReporter.calledCount, 0);
2496
2497 conf.set("hbasefsck.errorreporter", MockErrorReporter.class.getName());
2498 doFsck(conf, false);
2499 assertTrue(MockErrorReporter.calledCount > 20);
2500 } finally {
2501 conf.set("hbasefsck.errorreporter",
2502 PrintingErrorReporter.class.getName());
2503 MockErrorReporter.calledCount = 0;
2504 }
2505 }
2506
2507 static class MockErrorReporter implements ErrorReporter {
2508 static int calledCount = 0;
2509
2510 @Override
2511 public void clear() {
2512 calledCount++;
2513 }
2514
2515 @Override
2516 public void report(String message) {
2517 calledCount++;
2518 }
2519
2520 @Override
2521 public void reportError(String message) {
2522 calledCount++;
2523 }
2524
2525 @Override
2526 public void reportError(ERROR_CODE errorCode, String message) {
2527 calledCount++;
2528 }
2529
2530 @Override
2531 public void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
2532 calledCount++;
2533 }
2534
2535 @Override
2536 public void reportError(ERROR_CODE errorCode,
2537 String message, TableInfo table, HbckInfo info) {
2538 calledCount++;
2539 }
2540
2541 @Override
2542 public void reportError(ERROR_CODE errorCode, String message,
2543 TableInfo table, HbckInfo info1, HbckInfo info2) {
2544 calledCount++;
2545 }
2546
2547 @Override
2548 public int summarize() {
2549 return ++calledCount;
2550 }
2551
2552 @Override
2553 public void detail(String details) {
2554 calledCount++;
2555 }
2556
2557 @Override
2558 public ArrayList<ERROR_CODE> getErrorList() {
2559 calledCount++;
2560 return new ArrayList<ERROR_CODE>();
2561 }
2562
2563 @Override
2564 public void progress() {
2565 calledCount++;
2566 }
2567
2568 @Override
2569 public void print(String message) {
2570 calledCount++;
2571 }
2572
2573 @Override
2574 public void resetErrors() {
2575 calledCount++;
2576 }
2577
2578 @Override
2579 public boolean tableHasErrors(TableInfo table) {
2580 calledCount++;
2581 return false;
2582 }
2583 }
2584
2585 @Test(timeout=180000)
2586 public void testCheckTableLocks() throws Exception {
2587 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(0);
2588 EnvironmentEdgeManager.injectEdge(edge);
2589
2590 HBaseFsck hbck = doFsck(conf, false);
2591 assertNoErrors(hbck);
2592
2593 ServerName mockName = ServerName.valueOf("localhost", 60000, 1);
2594 final TableName tableName = TableName.valueOf("foo");
2595
2596
2597 final TableLockManager tableLockManager =
2598 TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName);
2599 TableLock writeLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
2600 writeLock.acquire();
2601 hbck = doFsck(conf, false);
2602 assertNoErrors(hbck);
2603
2604 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2605 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2606
2607 hbck = doFsck(conf, false);
2608 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2609
2610 final CountDownLatch latch = new CountDownLatch(1);
2611 new Thread() {
2612 @Override
2613 public void run() {
2614 TableLock readLock = tableLockManager.writeLock(tableName, "testCheckTableLocks");
2615 try {
2616 latch.countDown();
2617 readLock.acquire();
2618 } catch (IOException ex) {
2619 fail();
2620 } catch (IllegalStateException ex) {
2621 return;
2622 }
2623 fail("should not have come here");
2624 };
2625 }.start();
2626
2627 latch.await();
2628 Threads.sleep(300);
2629
2630 hbck = doFsck(conf, false);
2631 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2632
2633 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2634 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2635
2636 hbck = doFsck(conf, false);
2637 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK, ERROR_CODE.EXPIRED_TABLE_LOCK});
2638
2639 conf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
2640
2641 Threads.sleep(10);
2642 hbck = doFsck(conf, true);
2643
2644 hbck = doFsck(conf, false);
2645 assertNoErrors(hbck);
2646
2647
2648 writeLock = tableLockManager.writeLock(tableName, "should acquire without blocking");
2649 writeLock.acquire();
2650 writeLock.release();
2651 tableLockManager.tableDeleted(tableName);
2652 }
2653
2654
2655
2656
2657 @Test
2658 public void testOrphanedTableZNode() throws Exception {
2659 TableName table = TableName.valueOf("testOrphanedZKTableEntry");
2660
2661 try {
2662 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getTableStateManager()
2663 .setTableState(table, ZooKeeperProtos.Table.State.ENABLING);
2664
2665 try {
2666 setupTable(table);
2667 Assert.fail(
2668 "Create table should fail when its ZNode has already existed with ENABLING state.");
2669 } catch(TableExistsException t) {
2670
2671 }
2672
2673 try {
2674 cleanupTable(table);
2675 } catch (IOException e) {
2676
2677
2678 }
2679
2680 HBaseFsck hbck = doFsck(conf, false);
2681 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY));
2682
2683
2684 hbck = doFsck(conf, true);
2685
2686
2687 hbck = doFsck(conf, false);
2688 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY));
2689
2690 setupTable(table);
2691 } finally {
2692
2693
2694 try {
2695 cleanupTable(table);
2696 } catch (IOException e) {
2697
2698
2699 }
2700 }
2701 }
2702
2703 @Test (timeout=180000)
2704 public void testMetaOffline() throws Exception {
2705
2706 HBaseFsck hbck = doFsck(conf, false);
2707 assertNoErrors(hbck);
2708 deleteMetaRegion(conf, true, false, false);
2709 hbck = doFsck(conf, false);
2710
2711
2712 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2713 hbck = doFsck(conf, true);
2714 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2715 hbck = doFsck(conf, false);
2716 assertNoErrors(hbck);
2717 }
2718
2719 private void deleteMetaRegion(Configuration conf, boolean unassign, boolean hdfs,
2720 boolean regionInfoOnly) throws IOException, InterruptedException {
2721 HRegionLocation metaLocation = connection.getRegionLocator(TableName.META_TABLE_NAME)
2722 .getRegionLocation(HConstants.EMPTY_START_ROW);
2723 ServerName hsa = metaLocation.getServerName();
2724 HRegionInfo hri = metaLocation.getRegionInfo();
2725 if (unassign) {
2726 LOG.info("Undeploying meta region " + hri + " from server " + hsa);
2727 try (Connection unmanagedConnection = ConnectionFactory.createConnection(conf)) {
2728 undeployRegion(unmanagedConnection, hsa, hri);
2729 }
2730 }
2731
2732 if (regionInfoOnly) {
2733 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
2734 Path rootDir = FSUtils.getRootDir(conf);
2735 FileSystem fs = rootDir.getFileSystem(conf);
2736 Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
2737 hri.getEncodedName());
2738 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
2739 fs.delete(hriPath, true);
2740 }
2741
2742 if (hdfs) {
2743 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
2744 Path rootDir = FSUtils.getRootDir(conf);
2745 FileSystem fs = rootDir.getFileSystem(conf);
2746 Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
2747 hri.getEncodedName());
2748 HBaseFsck.debugLsr(conf, p);
2749 boolean success = fs.delete(p, true);
2750 LOG.info("Deleted " + p + " sucessfully? " + success);
2751 HBaseFsck.debugLsr(conf, p);
2752 }
2753 }
2754
2755 @Test (timeout=180000)
2756 public void testTableWithNoRegions() throws Exception {
2757
2758
2759 TableName table =
2760 TableName.valueOf(name.getMethodName());
2761 try {
2762
2763 HTableDescriptor desc = new HTableDescriptor(table);
2764 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
2765 desc.addFamily(hcd);
2766 createTable(TEST_UTIL, desc, null);
2767 tbl = (HTable) connection.getTable(table, tableExecutorService);
2768
2769
2770 deleteRegion(conf, tbl.getTableDescriptor(), HConstants.EMPTY_START_ROW,
2771 HConstants.EMPTY_END_ROW, false, false, true);
2772
2773 HBaseFsck hbck = doFsck(conf, false);
2774 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
2775
2776 doFsck(conf, true);
2777
2778
2779 doFsck(conf, true);
2780
2781
2782 assertNoErrors(doFsck(conf, false));
2783 } finally {
2784 cleanupTable(table);
2785 }
2786
2787 }
2788
2789 @Test (timeout=180000)
2790 public void testHbckAfterRegionMerge() throws Exception {
2791 TableName table = TableName.valueOf("testMergeRegionFilesInHdfs");
2792 Table meta = null;
2793 try {
2794
2795 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
2796 setupTable(table);
2797 assertEquals(ROWKEYS.length, countRows());
2798
2799
2800 admin.flush(table);
2801 HRegionInfo region1 = tbl.getRegionLocation(Bytes.toBytes("A")).getRegionInfo();
2802 HRegionInfo region2 = tbl.getRegionLocation(Bytes.toBytes("B")).getRegionInfo();
2803
2804 int regionCountBeforeMerge = tbl.getRegionLocations().size();
2805
2806 assertNotEquals(region1, region2);
2807
2808
2809 admin.mergeRegions(region1.getEncodedNameAsBytes(),
2810 region2.getEncodedNameAsBytes(), false);
2811
2812
2813 long timeout = System.currentTimeMillis() + 30 * 1000;
2814 while (true) {
2815 if (tbl.getRegionLocations().size() < regionCountBeforeMerge) {
2816 break;
2817 } else if (System.currentTimeMillis() > timeout) {
2818 fail("Time out waiting on region " + region1.getEncodedName()
2819 + " and " + region2.getEncodedName() + " be merged");
2820 }
2821 Thread.sleep(10);
2822 }
2823
2824 assertEquals(ROWKEYS.length, countRows());
2825
2826 HBaseFsck hbck = doFsck(conf, false);
2827 assertNoErrors(hbck);
2828
2829 } finally {
2830 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true);
2831 cleanupTable(table);
2832 IOUtils.closeQuietly(meta);
2833 }
2834 }
2835
2836 @Test (timeout = 180000)
2837 public void testRegionBoundariesCheck() throws Exception {
2838 HBaseFsck hbck = doFsck(conf, false);
2839 assertNoErrors(hbck);
2840 try {
2841 hbck.checkRegionBoundaries();
2842 } catch (IllegalArgumentException e) {
2843 if (e.getMessage().endsWith("not a valid DFS filename.")) {
2844 fail("Table directory path is not valid." + e.getMessage());
2845 }
2846 }
2847 }
2848
2849 @org.junit.Rule
2850 public TestName name = new TestName();
2851
2852 @Test (timeout=180000)
2853 public void testReadOnlyProperty() throws Exception {
2854 HBaseFsck hbck = doFsck(conf, false);
2855 Assert.assertEquals("shouldIgnorePreCheckPermission", true,
2856 hbck.shouldIgnorePreCheckPermission());
2857
2858 hbck = doFsck(conf, true);
2859 Assert.assertEquals("shouldIgnorePreCheckPermission", false,
2860 hbck.shouldIgnorePreCheckPermission());
2861
2862 hbck = doFsck(conf, true);
2863 hbck.setIgnorePreCheckPermission(true);
2864 Assert.assertEquals("shouldIgnorePreCheckPermission", true,
2865 hbck.shouldIgnorePreCheckPermission());
2866 }
2867
2868 @Test (timeout=180000)
2869 public void testCleanUpDaughtersNotInMetaAfterFailedSplit() throws Exception {
2870 TableName table = TableName.valueOf("testCleanUpDaughtersNotInMetaAfterFailedSplit");
2871 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
2872 try {
2873 HTableDescriptor desc = new HTableDescriptor(table);
2874 desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
2875 createTable(TEST_UTIL, desc, null);
2876 tbl = new HTable(cluster.getConfiguration(), desc.getTableName());
2877 for (int i = 0; i < 5; i++) {
2878 Put p1 = new Put(("r" + i).getBytes());
2879 p1.add(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes());
2880 tbl.put(p1);
2881 }
2882 admin.flush(desc.getTableName());
2883 List<HRegion> regions = cluster.getRegions(desc.getTableName());
2884 int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
2885 HRegionServer regionServer = cluster.getRegionServer(serverWith);
2886 cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
2887 SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
2888 st.prepare();
2889 st.stepsBeforePONR(regionServer, regionServer, false);
2890 AssignmentManager am = cluster.getMaster().getAssignmentManager();
2891 Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
2892 for (RegionState state : regionsInTransition.values()) {
2893 am.regionOffline(state.getRegion());
2894 }
2895 ZKAssign.deleteNodeFailSilent(regionServer.getZooKeeper(), regions.get(0).getRegionInfo());
2896 Map<HRegionInfo, ServerName> regionsMap = new HashMap<HRegionInfo, ServerName>();
2897 regionsMap.put(regions.get(0).getRegionInfo(), regionServer.getServerName());
2898 am.assign(regionsMap);
2899 am.waitForAssignment(regions.get(0).getRegionInfo());
2900 HBaseFsck hbck = doFsck(conf, false);
2901 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2902 ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
2903
2904 assertEquals(0, hbck.getOverlapGroups(table).size());
2905
2906
2907 assertErrors(
2908 doFsck(
2909 conf, false, true, false, false, false, false, false, false, false, false, false,
2910 false, null),
2911 new ERROR_CODE[] { ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
2912 ERROR_CODE.NOT_IN_META_OR_DEPLOYED });
2913
2914
2915 assertNoErrors(doFsck(conf, false));
2916 assertEquals(5, countRows());
2917 } finally {
2918 if (tbl != null) {
2919 tbl.close();
2920 tbl = null;
2921 }
2922 cleanupTable(table);
2923 }
2924 }
2925
2926
2927 public static class MasterSyncObserver extends BaseMasterObserver {
2928 volatile CountDownLatch tableCreationLatch = null;
2929 volatile CountDownLatch tableDeletionLatch = null;
2930
2931 @Override
2932 public void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
2933 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
2934
2935 if (tableCreationLatch != null) {
2936 tableCreationLatch.countDown();
2937 }
2938 }
2939
2940 @Override
2941 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
2942 TableName tableName)
2943 throws IOException {
2944
2945 if (tableDeletionLatch != null) {
2946 tableDeletionLatch.countDown();
2947 }
2948 }
2949 }
2950
2951 public static void createTable(HBaseTestingUtility testUtil, HTableDescriptor htd,
2952 byte [][] splitKeys) throws Exception {
2953
2954
2955 MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
2956 .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
2957 observer.tableCreationLatch = new CountDownLatch(1);
2958 if (splitKeys != null) {
2959 admin.createTable(htd, splitKeys);
2960 } else {
2961 admin.createTable(htd);
2962 }
2963 observer.tableCreationLatch.await();
2964 observer.tableCreationLatch = null;
2965 testUtil.waitUntilAllRegionsAssigned(htd.getTableName());
2966 }
2967
2968 public static void deleteTable(HBaseTestingUtility testUtil, TableName tableName)
2969 throws Exception {
2970
2971
2972 MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
2973 .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
2974 observer.tableDeletionLatch = new CountDownLatch(1);
2975 try {
2976 admin.disableTable(tableName);
2977 } catch (Exception e) {
2978 LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
2979 }
2980 admin.deleteTable(tableName);
2981 observer.tableDeletionLatch.await();
2982 observer.tableDeletionLatch = null;
2983 }
2984 }