1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import java.io.Closeable;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.io.PrintWriter;
25 import java.io.StringWriter;
26 import java.net.InetAddress;
27 import java.net.URI;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.Collections;
32 import java.util.Comparator;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.Iterator;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.Set;
40 import java.util.SortedMap;
41 import java.util.SortedSet;
42 import java.util.TreeMap;
43 import java.util.TreeSet;
44 import java.util.concurrent.Callable;
45 import java.util.concurrent.ConcurrentSkipListMap;
46 import java.util.concurrent.ExecutionException;
47 import java.util.concurrent.ExecutorService;
48 import java.util.concurrent.Executors;
49 import java.util.concurrent.Future;
50 import java.util.concurrent.FutureTask;
51 import java.util.concurrent.ScheduledThreadPoolExecutor;
52 import java.util.concurrent.TimeUnit;
53 import java.util.concurrent.TimeoutException;
54 import java.util.concurrent.atomic.AtomicBoolean;
55 import java.util.concurrent.atomic.AtomicInteger;
56
57 import org.apache.commons.io.IOUtils;
58 import org.apache.commons.lang.StringUtils;
59 import org.apache.commons.logging.Log;
60 import org.apache.commons.logging.LogFactory;
61 import org.apache.hadoop.conf.Configuration;
62 import org.apache.hadoop.conf.Configured;
63 import org.apache.hadoop.fs.FSDataOutputStream;
64 import org.apache.hadoop.fs.FileStatus;
65 import org.apache.hadoop.fs.FileSystem;
66 import org.apache.hadoop.fs.Path;
67 import org.apache.hadoop.fs.permission.FsAction;
68 import org.apache.hadoop.fs.permission.FsPermission;
69 import org.apache.hadoop.hbase.Abortable;
70 import org.apache.hadoop.hbase.Cell;
71 import org.apache.hadoop.hbase.ClusterStatus;
72 import org.apache.hadoop.hbase.CoordinatedStateException;
73 import org.apache.hadoop.hbase.HBaseConfiguration;
74 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
75 import org.apache.hadoop.hbase.HColumnDescriptor;
76 import org.apache.hadoop.hbase.HConstants;
77 import org.apache.hadoop.hbase.HRegionInfo;
78 import org.apache.hadoop.hbase.HRegionLocation;
79 import org.apache.hadoop.hbase.HTableDescriptor;
80 import org.apache.hadoop.hbase.KeyValue;
81 import org.apache.hadoop.hbase.MasterNotRunningException;
82 import org.apache.hadoop.hbase.MetaTableAccessor;
83 import org.apache.hadoop.hbase.RegionLocations;
84 import org.apache.hadoop.hbase.ServerName;
85 import org.apache.hadoop.hbase.TableName;
86 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
87 import org.apache.hadoop.hbase.classification.InterfaceAudience;
88 import org.apache.hadoop.hbase.classification.InterfaceStability;
89 import org.apache.hadoop.hbase.client.Admin;
90 import org.apache.hadoop.hbase.client.ClusterConnection;
91 import org.apache.hadoop.hbase.client.ConnectionFactory;
92 import org.apache.hadoop.hbase.client.Delete;
93 import org.apache.hadoop.hbase.client.Get;
94 import org.apache.hadoop.hbase.client.HBaseAdmin;
95 import org.apache.hadoop.hbase.client.HConnectable;
96 import org.apache.hadoop.hbase.client.HConnection;
97 import org.apache.hadoop.hbase.client.HConnectionManager;
98 import org.apache.hadoop.hbase.client.MetaScanner;
99 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
100 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
101 import org.apache.hadoop.hbase.client.Put;
102 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
103 import org.apache.hadoop.hbase.client.Result;
104 import org.apache.hadoop.hbase.client.RowMutations;
105 import org.apache.hadoop.hbase.client.Table;
106 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
107 import org.apache.hadoop.hbase.io.hfile.HFile;
108 import org.apache.hadoop.hbase.master.MasterFileSystem;
109 import org.apache.hadoop.hbase.master.RegionState;
110 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
111 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
112 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
113 import org.apache.hadoop.hbase.regionserver.HRegion;
114 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
115 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
116 import org.apache.hadoop.hbase.security.AccessDeniedException;
117 import org.apache.hadoop.hbase.security.UserProvider;
118 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
119 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
120 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
121 import org.apache.hadoop.hbase.util.hbck.ReplicationChecker;
122 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
123 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
124 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
125 import org.apache.hadoop.hbase.wal.WALSplitter;
126 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
127 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
128 import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager;
129 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
130 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
131 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
132 import org.apache.hadoop.ipc.RemoteException;
133 import org.apache.hadoop.security.UserGroupInformation;
134 import org.apache.hadoop.util.ReflectionUtils;
135 import org.apache.hadoop.util.Tool;
136 import org.apache.hadoop.util.ToolRunner;
137 import org.apache.zookeeper.KeeperException;
138
139 import com.google.common.base.Joiner;
140 import com.google.common.base.Preconditions;
141 import com.google.common.collect.ImmutableList;
142 import com.google.common.collect.Lists;
143 import com.google.common.collect.Multimap;
144 import com.google.common.collect.Ordering;
145 import com.google.common.collect.TreeMultimap;
146 import com.google.protobuf.ServiceException;
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
194 @InterfaceStability.Evolving
195 public class HBaseFsck extends Configured implements Closeable {
196 public static final long DEFAULT_TIME_LAG = 60000;
197 public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
198 private static final int MAX_NUM_THREADS = 50;
199 private static boolean rsSupportsOffline = true;
200 private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
201 private static final int DEFAULT_MAX_MERGE = 5;
202 private static final String TO_BE_LOADED = "to_be_loaded";
203 private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
204 private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
205 private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200;
206 private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000;
207
208
209
210
211 private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80;
212
213
214
215
216 private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
217 private ClusterStatus status;
218 private ClusterConnection connection;
219 private Admin admin;
220 private Table meta;
221
222 protected ExecutorService executor;
223 private long startMillis = EnvironmentEdgeManager.currentTime();
224 private HFileCorruptionChecker hfcc;
225 private int retcode = 0;
226 private Path HBCK_LOCK_PATH;
227 private FSDataOutputStream hbckOutFd;
228
229
230
231 private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
232
233
234
235
236 private static boolean details = false;
237 private long timelag = DEFAULT_TIME_LAG;
238 private static boolean forceExclusive = false;
239 private static boolean disableBalancer = false;
240 private boolean fixAssignments = false;
241 private boolean fixMeta = false;
242 private boolean checkHdfs = true;
243 private boolean fixHdfsHoles = false;
244 private boolean fixHdfsOverlaps = false;
245 private boolean fixHdfsOrphans = false;
246 private boolean fixTableOrphans = false;
247 private boolean fixVersionFile = false;
248 private boolean fixSplitParents = false;
249 private boolean fixReferenceFiles = false;
250 private boolean fixEmptyMetaCells = false;
251 private boolean fixTableLocks = false;
252 private boolean fixTableZNodes = false;
253 private boolean fixReplication = false;
254 private boolean fixAny = false;
255
256
257
258 private Set<TableName> tablesIncluded = new HashSet<TableName>();
259 private int maxMerge = DEFAULT_MAX_MERGE;
260 private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
261 private boolean sidelineBigOverlaps = false;
262 private Path sidelineDir = null;
263
264 private boolean rerun = false;
265 private static boolean SUMMARY = false;
266 private boolean checkMetaOnly = false;
267 private boolean checkRegionBoundaries = false;
268 private boolean ignorePreCheckPermission = false;
269
270
271
272
273 final private ErrorReporter errors;
274 int fixes = 0;
275
276
277
278
279
280
281 private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
282 private TreeSet<TableName> disabledTables =
283 new TreeSet<TableName>();
284
285 private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
286
287
288
289
290
291
292
293
294
295
296
297 private SortedMap<TableName, TableInfo> tablesInfo =
298 new ConcurrentSkipListMap<TableName, TableInfo>();
299
300
301
302
303 private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
304
305 private Map<TableName, Set<String>> orphanTableDirs =
306 new HashMap<TableName, Set<String>>();
307
308 private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
309
310
311
312
313 private Set<TableName> orphanedTableZNodes = new HashSet<TableName>();
314 private final RetryCounterFactory lockFileRetryCounterFactory;
315
316
317
318
319
320
321
322
323
324 public HBaseFsck(Configuration conf) throws MasterNotRunningException,
325 ZooKeeperConnectionException, IOException, ClassNotFoundException {
326 this(conf, createThreadPool(conf));
327 }
328
329 private static ExecutorService createThreadPool(Configuration conf) {
330 int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
331 return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
332 }
333
334
335
336
337
338
339
340
341
342
343
344 public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
345 ZooKeeperConnectionException, IOException, ClassNotFoundException {
346 super(conf);
347 errors = getErrorReporter(getConf());
348 this.executor = exec;
349 lockFileRetryCounterFactory = new RetryCounterFactory(
350 getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
351 getConf().getInt(
352 "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
353 getConf().getInt(
354 "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
355 }
356
357 private class FileLockCallable implements Callable<FSDataOutputStream> {
358 RetryCounter retryCounter;
359
360 public FileLockCallable(RetryCounter retryCounter) {
361 this.retryCounter = retryCounter;
362 }
363 @Override
364 public FSDataOutputStream call() throws IOException {
365 try {
366 FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
367 FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
368 HConstants.DATA_FILE_UMASK_KEY);
369 Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
370 fs.mkdirs(tmpDir);
371 HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
372 final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms);
373 out.writeBytes(InetAddress.getLocalHost().toString());
374 out.flush();
375 return out;
376 } catch(RemoteException e) {
377 if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
378 return null;
379 } else {
380 throw e;
381 }
382 }
383 }
384
385 private FSDataOutputStream createFileWithRetries(final FileSystem fs,
386 final Path hbckLockFilePath, final FsPermission defaultPerms)
387 throws IOException {
388
389 IOException exception = null;
390 do {
391 try {
392 return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
393 } catch (IOException ioe) {
394 LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
395 + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
396 + retryCounter.getMaxAttempts());
397 LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
398 ioe);
399 try {
400 exception = ioe;
401 retryCounter.sleepUntilNextRetry();
402 } catch (InterruptedException ie) {
403 throw (InterruptedIOException) new InterruptedIOException(
404 "Can't create lock file " + hbckLockFilePath.getName())
405 .initCause(ie);
406 }
407 }
408 } while (retryCounter.shouldRetry());
409
410 throw exception;
411 }
412 }
413
414
415
416
417
418
419
420 private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
421 RetryCounter retryCounter = lockFileRetryCounterFactory.create();
422 FileLockCallable callable = new FileLockCallable(retryCounter);
423 ExecutorService executor = Executors.newFixedThreadPool(1);
424 FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
425 executor.execute(futureTask);
426 final int timeoutInSeconds = getConf().getInt(
427 "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
428 FSDataOutputStream stream = null;
429 try {
430 stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
431 } catch (ExecutionException ee) {
432 LOG.warn("Encountered exception when opening lock file", ee);
433 } catch (InterruptedException ie) {
434 LOG.warn("Interrupted when opening lock file", ie);
435 Thread.currentThread().interrupt();
436 } catch (TimeoutException exception) {
437
438 LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
439 futureTask.cancel(true);
440 } finally {
441 executor.shutdownNow();
442 }
443 return stream;
444 }
445
446 private void unlockHbck() {
447 if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
448 RetryCounter retryCounter = lockFileRetryCounterFactory.create();
449 do {
450 try {
451 IOUtils.closeQuietly(hbckOutFd);
452 FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()),
453 HBCK_LOCK_PATH, true);
454 LOG.info("Finishing hbck");
455 return;
456 } catch (IOException ioe) {
457 LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
458 + (retryCounter.getAttemptTimes() + 1) + " of "
459 + retryCounter.getMaxAttempts());
460 LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
461 try {
462 retryCounter.sleepUntilNextRetry();
463 } catch (InterruptedException ie) {
464 Thread.currentThread().interrupt();
465 LOG.warn("Interrupted while deleting lock file" +
466 HBCK_LOCK_PATH);
467 return;
468 }
469 }
470 } while (retryCounter.shouldRetry());
471 }
472 }
473
474
475
476
477
478 public void connect() throws IOException {
479
480 if (isExclusive()) {
481
482 hbckOutFd = checkAndMarkRunningHbck();
483 if (hbckOutFd == null) {
484 setRetCode(-1);
485 LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
486 "[If you are sure no other instance is running, delete the lock file " +
487 HBCK_LOCK_PATH + " and rerun the tool]");
488 throw new IOException("Duplicate hbck - Abort");
489 }
490
491
492 hbckLockCleanup.set(true);
493 }
494
495
496
497
498
499 Runtime.getRuntime().addShutdownHook(new Thread() {
500 @Override
501 public void run() {
502 IOUtils.closeQuietly(HBaseFsck.this);
503 unlockHbck();
504 }
505 });
506
507 LOG.info("Launching hbck");
508
509 connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
510 admin = connection.getAdmin();
511 meta = connection.getTable(TableName.META_TABLE_NAME);
512 status = admin.getClusterStatus();
513 }
514
515
516
517
518 private void loadDeployedRegions() throws IOException, InterruptedException {
519
520 Collection<ServerName> regionServers = status.getServers();
521 errors.print("Number of live region servers: " + regionServers.size());
522 if (details) {
523 for (ServerName rsinfo: regionServers) {
524 errors.print(" " + rsinfo.getServerName());
525 }
526 }
527
528
529 Collection<ServerName> deadRegionServers = status.getDeadServerNames();
530 errors.print("Number of dead region servers: " + deadRegionServers.size());
531 if (details) {
532 for (ServerName name: deadRegionServers) {
533 errors.print(" " + name);
534 }
535 }
536
537
538 errors.print("Master: " + status.getMaster());
539
540
541 Collection<ServerName> backupMasters = status.getBackupMasters();
542 errors.print("Number of backup masters: " + backupMasters.size());
543 if (details) {
544 for (ServerName name: backupMasters) {
545 errors.print(" " + name);
546 }
547 }
548
549 errors.print("Average load: " + status.getAverageLoad());
550 errors.print("Number of requests: " + status.getRequestsCount());
551 errors.print("Number of regions: " + status.getRegionsCount());
552
553 Map<String, RegionState> rits = status.getRegionsInTransition();
554 errors.print("Number of regions in transition: " + rits.size());
555 if (details) {
556 for (RegionState state: rits.values()) {
557 errors.print(" " + state.toDescriptiveString());
558 }
559 }
560
561
562 processRegionServers(regionServers);
563 }
564
565
566
567
568 private void clearState() {
569
570 fixes = 0;
571 regionInfoMap.clear();
572 emptyRegionInfoQualifiers.clear();
573 disabledTables.clear();
574 errors.clear();
575 tablesInfo.clear();
576 orphanHdfsDirs.clear();
577 skippedRegions.clear();
578 }
579
580
581
582
583
584
585 public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
586
587 if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
588 || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
589 LOG.info("Loading regioninfos HDFS");
590
591 int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
592 int curIter = 0;
593 do {
594 clearState();
595
596 restoreHdfsIntegrity();
597 curIter++;
598 } while (fixes > 0 && curIter <= maxIterations);
599
600
601
602 if (curIter > 2) {
603 if (curIter == maxIterations) {
604 LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
605 + "Tables integrity may not be fully repaired!");
606 } else {
607 LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
608 }
609 }
610 }
611 }
612
613
614
615
616
617
618
619
620
621 public int onlineConsistencyRepair() throws IOException, KeeperException,
622 InterruptedException {
623 clearState();
624
625
626 loadDeployedRegions();
627
628 recordMetaRegion();
629
630 if (!checkMetaRegion()) {
631 String errorMsg = "hbase:meta table is not consistent. ";
632 if (shouldFixAssignments()) {
633 errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
634 } else {
635 errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
636 }
637 errors.reportError(errorMsg + " Exiting...");
638 return -2;
639 }
640
641 LOG.info("Loading regionsinfo from the hbase:meta table");
642 boolean success = loadMetaEntries();
643 if (!success) return -1;
644
645
646 reportEmptyMetaCells();
647
648
649 if (shouldFixEmptyMetaCells()) {
650 fixEmptyMetaCells();
651 }
652
653
654 if (!checkMetaOnly) {
655 reportTablesInFlux();
656 }
657
658
659 if (shouldCheckHdfs()) {
660 LOG.info("Loading region directories from HDFS");
661 loadHdfsRegionDirs();
662 LOG.info("Loading region information from HDFS");
663 loadHdfsRegionInfos();
664 }
665
666
667 loadDisabledTables();
668
669
670 fixOrphanTables();
671
672 LOG.info("Checking and fixing region consistency");
673
674 checkAndFixConsistency();
675
676
677 checkIntegrity();
678 return errors.getErrorList().size();
679 }
680
681
682
683
684
685 public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
686
687 errors.print("Version: " + status.getHBaseVersion());
688 offlineHdfsIntegrityRepair();
689
690 boolean oldBalancer = false;
691 if (shouldDisableBalancer()) {
692 oldBalancer = admin.setBalancerRunning(false, true);
693 }
694
695 try {
696 onlineConsistencyRepair();
697 }
698 finally {
699
700
701
702 if (shouldDisableBalancer() && oldBalancer) {
703 admin.setBalancerRunning(oldBalancer, false);
704 }
705 }
706
707 if (checkRegionBoundaries) {
708 checkRegionBoundaries();
709 }
710
711 offlineReferenceFileRepair();
712
713 checkAndFixTableLocks();
714
715
716 checkAndFixOrphanedTableZNodes();
717
718 checkAndFixReplication();
719
720
721 unlockHbck();
722
723
724 printTableSummary(tablesInfo);
725 return errors.summarize();
726 }
727
728 public static byte[] keyOnly (byte[] b) {
729 if (b == null)
730 return b;
731 int rowlength = Bytes.toShort(b, 0);
732 byte[] result = new byte[rowlength];
733 System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
734 return result;
735 }
736
737 @Override
738 public void close() throws IOException {
739 IOUtils.closeQuietly(admin);
740 IOUtils.closeQuietly(meta);
741 IOUtils.closeQuietly(connection);
742 }
743
744 private static class RegionBoundariesInformation {
745 public byte [] regionName;
746 public byte [] metaFirstKey;
747 public byte [] metaLastKey;
748 public byte [] storesFirstKey;
749 public byte [] storesLastKey;
750 @Override
751 public String toString () {
752 return "regionName=" + Bytes.toStringBinary(regionName) +
753 "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
754 "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
755 "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
756 "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
757 }
758 }
759
760 public void checkRegionBoundaries() {
761 try {
762 ByteArrayComparator comparator = new ByteArrayComparator();
763 List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), connection, false);
764 final RegionBoundariesInformation currentRegionBoundariesInformation =
765 new RegionBoundariesInformation();
766 Path hbaseRoot = FSUtils.getRootDir(getConf());
767 for (HRegionInfo regionInfo : regions) {
768 Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
769 currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
770
771
772 Path path = new Path(tableDir, regionInfo.getEncodedName());
773 FileSystem fs = path.getFileSystem(getConf());
774 FileStatus[] files = fs.listStatus(path);
775
776 byte[] storeFirstKey = null;
777 byte[] storeLastKey = null;
778 for (FileStatus file : files) {
779 String fileName = file.getPath().toString();
780 fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
781 if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
782 FileStatus[] storeFiles = fs.listStatus(file.getPath());
783
784 for (FileStatus storeFile : storeFiles) {
785 HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
786 getConf()), getConf());
787 if ((reader.getFirstKey() != null)
788 && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
789 reader.getFirstKey()) > 0))) {
790 storeFirstKey = reader.getFirstKey();
791 }
792 if ((reader.getLastKey() != null)
793 && ((storeLastKey == null) || (comparator.compare(storeLastKey,
794 reader.getLastKey())) < 0)) {
795 storeLastKey = reader.getLastKey();
796 }
797 reader.close();
798 }
799 }
800 }
801 currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
802 currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
803 currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
804 currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
805 if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
806 currentRegionBoundariesInformation.metaFirstKey = null;
807 if (currentRegionBoundariesInformation.metaLastKey.length == 0)
808 currentRegionBoundariesInformation.metaLastKey = null;
809
810
811
812
813
814
815 boolean valid = true;
816
817 if ((currentRegionBoundariesInformation.storesFirstKey != null)
818 && (currentRegionBoundariesInformation.metaFirstKey != null)) {
819 valid = valid
820 && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
821 currentRegionBoundariesInformation.metaFirstKey) >= 0;
822 }
823
824 if ((currentRegionBoundariesInformation.storesLastKey != null)
825 && (currentRegionBoundariesInformation.metaLastKey != null)) {
826 valid = valid
827 && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
828 currentRegionBoundariesInformation.metaLastKey) < 0;
829 }
830 if (!valid) {
831 errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
832 tablesInfo.get(regionInfo.getTable()));
833 LOG.warn("Region's boundaries not alligned between stores and META for:");
834 LOG.warn(currentRegionBoundariesInformation);
835 }
836 }
837 } catch (IOException e) {
838 LOG.error(e);
839 }
840 }
841
842
843
844
845 private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
846 for (HbckInfo hi : orphanHdfsDirs) {
847 LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
848 adoptHdfsOrphan(hi);
849 }
850 }
851
852
853
854
855
856
857
858
859
860
861 @SuppressWarnings("deprecation")
862 private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
863 Path p = hi.getHdfsRegionDir();
864 FileSystem fs = p.getFileSystem(getConf());
865 FileStatus[] dirs = fs.listStatus(p);
866 if (dirs == null) {
867 LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
868 p + ". This dir could probably be deleted.");
869 return ;
870 }
871
872 TableName tableName = hi.getTableName();
873 TableInfo tableInfo = tablesInfo.get(tableName);
874 Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
875 HTableDescriptor template = tableInfo.getHTD();
876
877
878 Pair<byte[],byte[]> orphanRegionRange = null;
879 for (FileStatus cf : dirs) {
880 String cfName= cf.getPath().getName();
881
882 if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
883
884 FileStatus[] hfiles = fs.listStatus(cf.getPath());
885 for (FileStatus hfile : hfiles) {
886 byte[] start, end;
887 HFile.Reader hf = null;
888 try {
889 CacheConfig cacheConf = new CacheConfig(getConf());
890 hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
891 hf.loadFileInfo();
892 KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
893 start = startKv.getRow();
894 KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
895 end = endKv.getRow();
896 } catch (IOException ioe) {
897 LOG.warn("Problem reading orphan file " + hfile + ", skipping");
898 continue;
899 } catch (NullPointerException ioe) {
900 LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
901 continue;
902 } finally {
903 if (hf != null) {
904 hf.close();
905 }
906 }
907
908
909 if (orphanRegionRange == null) {
910
911 orphanRegionRange = new Pair<byte[], byte[]>(start, end);
912 } else {
913
914
915
916 if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
917 orphanRegionRange.setFirst(start);
918 }
919 if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
920 orphanRegionRange.setSecond(end);
921 }
922 }
923 }
924 }
925 if (orphanRegionRange == null) {
926 LOG.warn("No data in dir " + p + ", sidelining data");
927 fixes++;
928 sidelineRegionDir(fs, hi);
929 return;
930 }
931 LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
932 Bytes.toString(orphanRegionRange.getSecond()) + ")");
933
934
935 HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
936 LOG.info("Creating new region : " + hri);
937 HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
938 Path target = region.getRegionFileSystem().getRegionDir();
939
940
941 mergeRegionDirs(target, hi);
942 fixes++;
943 }
944
945
946
947
948
949
950
951
952
953 private int restoreHdfsIntegrity() throws IOException, InterruptedException {
954
955 LOG.info("Loading HBase regioninfo from HDFS...");
956 loadHdfsRegionDirs();
957
958 int errs = errors.getErrorList().size();
959
960 tablesInfo = loadHdfsRegionInfos();
961 checkHdfsIntegrity(false, false);
962
963 if (errors.getErrorList().size() == errs) {
964 LOG.info("No integrity errors. We are done with this phase. Glorious.");
965 return 0;
966 }
967
968 if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
969 adoptHdfsOrphans(orphanHdfsDirs);
970
971 }
972
973
974 if (shouldFixHdfsHoles()) {
975 clearState();
976 loadHdfsRegionDirs();
977 tablesInfo = loadHdfsRegionInfos();
978 tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
979 }
980
981
982 if (shouldFixHdfsOverlaps()) {
983
984 clearState();
985 loadHdfsRegionDirs();
986 tablesInfo = loadHdfsRegionInfos();
987 tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
988 }
989
990 return errors.getErrorList().size();
991 }
992
993
994
995
996
997
998
999
1000
1001 private void offlineReferenceFileRepair() throws IOException {
1002 Configuration conf = getConf();
1003 Path hbaseRoot = FSUtils.getRootDir(conf);
1004 FileSystem fs = hbaseRoot.getFileSystem(conf);
1005 LOG.info("Computing mapping of all store files");
1006 Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors);
1007 errors.print("");
1008 LOG.info("Validating mapping using HDFS state");
1009 for (Path path: allFiles.values()) {
1010 boolean isReference = false;
1011 try {
1012 isReference = StoreFileInfo.isReference(path);
1013 } catch (Throwable t) {
1014
1015
1016
1017
1018 }
1019 if (!isReference) continue;
1020
1021 Path referredToFile = StoreFileInfo.getReferredToFile(path);
1022 if (fs.exists(referredToFile)) continue;
1023
1024
1025 errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1026 "Found lingering reference file " + path);
1027 if (!shouldFixReferenceFiles()) continue;
1028
1029
1030 boolean success = false;
1031 String pathStr = path.toString();
1032
1033
1034
1035
1036
1037 int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1038 for (int i = 0; index > 0 && i < 5; i++) {
1039 index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1040 }
1041 if (index > 0) {
1042 Path rootDir = getSidelineDir();
1043 Path dst = new Path(rootDir, pathStr.substring(index + 1));
1044 fs.mkdirs(dst.getParent());
1045 LOG.info("Trying to sildeline reference file "
1046 + path + " to " + dst);
1047 setShouldRerun();
1048
1049 success = fs.rename(path, dst);
1050 }
1051 if (!success) {
1052 LOG.error("Failed to sideline reference file " + path);
1053 }
1054 }
1055 }
1056
1057
1058
1059
1060 private void reportEmptyMetaCells() {
1061 errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1062 emptyRegionInfoQualifiers.size());
1063 if (details) {
1064 for (Result r: emptyRegionInfoQualifiers) {
1065 errors.print(" " + r);
1066 }
1067 }
1068 }
1069
1070
1071
1072
1073 private void reportTablesInFlux() {
1074 AtomicInteger numSkipped = new AtomicInteger(0);
1075 HTableDescriptor[] allTables = getTables(numSkipped);
1076 errors.print("Number of Tables: " + allTables.length);
1077 if (details) {
1078 if (numSkipped.get() > 0) {
1079 errors.detail("Number of Tables in flux: " + numSkipped.get());
1080 }
1081 for (HTableDescriptor td : allTables) {
1082 errors.detail(" Table: " + td.getTableName() + "\t" +
1083 (td.isReadOnly() ? "ro" : "rw") + "\t" +
1084 (td.isMetaRegion() ? "META" : " ") + "\t" +
1085 " families: " + td.getFamilies().size());
1086 }
1087 }
1088 }
1089
1090 public ErrorReporter getErrors() {
1091 return errors;
1092 }
1093
1094
1095
1096
1097
1098 private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
1099 Path regionDir = hbi.getHdfsRegionDir();
1100 if (regionDir == null) {
1101 LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1102 return;
1103 }
1104
1105 if (hbi.hdfsEntry.hri != null) {
1106
1107 return;
1108 }
1109
1110 FileSystem fs = FileSystem.get(getConf());
1111 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1112 LOG.debug("HRegionInfo read: " + hri.toString());
1113 hbi.hdfsEntry.hri = hri;
1114 }
1115
1116
1117
1118
1119
1120 public static class RegionRepairException extends IOException {
1121 private static final long serialVersionUID = 1L;
1122 final IOException ioe;
1123 public RegionRepairException(String s, IOException ioe) {
1124 super(s);
1125 this.ioe = ioe;
1126 }
1127 }
1128
1129
1130
1131
1132 private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1133 throws IOException, InterruptedException {
1134 tablesInfo.clear();
1135
1136 Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1137
1138
1139 List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1140 List<Future<Void>> hbiFutures;
1141
1142 for (HbckInfo hbi : hbckInfos) {
1143 WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1144 hbis.add(work);
1145 }
1146
1147
1148 hbiFutures = executor.invokeAll(hbis);
1149
1150 for(int i=0; i<hbiFutures.size(); i++) {
1151 WorkItemHdfsRegionInfo work = hbis.get(i);
1152 Future<Void> f = hbiFutures.get(i);
1153 try {
1154 f.get();
1155 } catch(ExecutionException e) {
1156 LOG.warn("Failed to read .regioninfo file for region " +
1157 work.hbi.getRegionNameAsString(), e.getCause());
1158 }
1159 }
1160
1161 Path hbaseRoot = FSUtils.getRootDir(getConf());
1162 FileSystem fs = hbaseRoot.getFileSystem(getConf());
1163
1164 for (HbckInfo hbi: hbckInfos) {
1165
1166 if (hbi.getHdfsHRI() == null) {
1167
1168 continue;
1169 }
1170
1171
1172
1173 TableName tableName = hbi.getTableName();
1174 if (tableName == null) {
1175
1176 LOG.warn("tableName was null for: " + hbi);
1177 continue;
1178 }
1179
1180 TableInfo modTInfo = tablesInfo.get(tableName);
1181 if (modTInfo == null) {
1182
1183 modTInfo = new TableInfo(tableName);
1184 tablesInfo.put(tableName, modTInfo);
1185 try {
1186 HTableDescriptor htd =
1187 FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1188 modTInfo.htds.add(htd);
1189 } catch (IOException ioe) {
1190 if (!orphanTableDirs.containsKey(tableName)) {
1191 LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1192
1193 errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1194 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1195 Set<String> columns = new HashSet<String>();
1196 orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1197 }
1198 }
1199 }
1200 if (!hbi.isSkipChecks()) {
1201 modTInfo.addRegionInfo(hbi);
1202 }
1203 }
1204
1205 loadTableInfosForTablesWithNoRegion();
1206 errors.print("");
1207
1208 return tablesInfo;
1209 }
1210
1211
1212
1213
1214
1215
1216
1217
1218 private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1219 Path regionDir = hbi.getHdfsRegionDir();
1220 FileSystem fs = regionDir.getFileSystem(getConf());
1221 FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1222 for (FileStatus subdir : subDirs) {
1223 String columnfamily = subdir.getPath().getName();
1224 columns.add(columnfamily);
1225 }
1226 return columns;
1227 }
1228
1229
1230
1231
1232
1233
1234
1235
1236 private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1237 Set<String> columns) throws IOException {
1238 if (columns ==null || columns.isEmpty()) return false;
1239 HTableDescriptor htd = new HTableDescriptor(tableName);
1240 for (String columnfamimly : columns) {
1241 htd.addFamily(new HColumnDescriptor(columnfamimly));
1242 }
1243 fstd.createTableDescriptor(htd, true);
1244 return true;
1245 }
1246
1247
1248
1249
1250
1251 public void fixEmptyMetaCells() throws IOException {
1252 if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1253 LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1254 for (Result region : emptyRegionInfoQualifiers) {
1255 deleteMetaRegion(region.getRow());
1256 errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1257 }
1258 emptyRegionInfoQualifiers.clear();
1259 }
1260 }
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271 public void fixOrphanTables() throws IOException {
1272 if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1273
1274 List<TableName> tmpList = new ArrayList<TableName>();
1275 tmpList.addAll(orphanTableDirs.keySet());
1276 HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1277 Iterator<Entry<TableName, Set<String>>> iter =
1278 orphanTableDirs.entrySet().iterator();
1279 int j = 0;
1280 int numFailedCase = 0;
1281 FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1282 while (iter.hasNext()) {
1283 Entry<TableName, Set<String>> entry =
1284 iter.next();
1285 TableName tableName = entry.getKey();
1286 LOG.info("Trying to fix orphan table error: " + tableName);
1287 if (j < htds.length) {
1288 if (tableName.equals(htds[j].getTableName())) {
1289 HTableDescriptor htd = htds[j];
1290 LOG.info("fixing orphan table: " + tableName + " from cache");
1291 fstd.createTableDescriptor(htd, true);
1292 j++;
1293 iter.remove();
1294 }
1295 } else {
1296 if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1297 LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1298 LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1299 iter.remove();
1300 } else {
1301 LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1302 numFailedCase++;
1303 }
1304 }
1305 fixes++;
1306 }
1307
1308 if (orphanTableDirs.isEmpty()) {
1309
1310
1311 setShouldRerun();
1312 LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1313 } else if (numFailedCase > 0) {
1314 LOG.error("Failed to fix " + numFailedCase
1315 + " OrphanTables with default .tableinfo files");
1316 }
1317
1318 }
1319
1320 orphanTableDirs.clear();
1321
1322 }
1323
1324
1325
1326
1327
1328
1329 private HRegion createNewMeta() throws IOException {
1330 Path rootdir = FSUtils.getRootDir(getConf());
1331 Configuration c = getConf();
1332 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1333 HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1334 MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1335 HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor);
1336 MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1337 return meta;
1338 }
1339
1340
1341
1342
1343
1344
1345
1346 private ArrayList<Put> generatePuts(
1347 SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1348 ArrayList<Put> puts = new ArrayList<Put>();
1349 boolean hasProblems = false;
1350 for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1351 TableName name = e.getKey();
1352
1353
1354 if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1355 continue;
1356 }
1357
1358 TableInfo ti = e.getValue();
1359 for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1360 .entrySet()) {
1361 Collection<HbckInfo> his = spl.getValue();
1362 int sz = his.size();
1363 if (sz != 1) {
1364
1365 LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1366 + " had " + sz + " regions instead of exactly 1." );
1367 hasProblems = true;
1368 continue;
1369 }
1370
1371
1372 HbckInfo hi = his.iterator().next();
1373 HRegionInfo hri = hi.getHdfsHRI();
1374 Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1375 puts.add(p);
1376 }
1377 }
1378 return hasProblems ? null : puts;
1379 }
1380
1381
1382
1383
1384 private void suggestFixes(
1385 SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1386 logParallelMerge();
1387 for (TableInfo tInfo : tablesInfo.values()) {
1388 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1389 tInfo.checkRegionChain(handler);
1390 }
1391 }
1392
1393
1394
1395
1396
1397
1398
1399
1400 public boolean rebuildMeta(boolean fix) throws IOException,
1401 InterruptedException {
1402
1403
1404
1405
1406
1407 LOG.info("Loading HBase regioninfo from HDFS...");
1408 loadHdfsRegionDirs();
1409
1410 int errs = errors.getErrorList().size();
1411 tablesInfo = loadHdfsRegionInfos();
1412 checkHdfsIntegrity(false, false);
1413
1414
1415 if (errors.getErrorList().size() != errs) {
1416
1417 while(true) {
1418 fixes = 0;
1419 suggestFixes(tablesInfo);
1420 errors.clear();
1421 loadHdfsRegionInfos();
1422 checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1423
1424 int errCount = errors.getErrorList().size();
1425
1426 if (fixes == 0) {
1427 if (errCount > 0) {
1428 return false;
1429 } else {
1430 break;
1431 }
1432 }
1433 }
1434 }
1435
1436
1437 LOG.info("HDFS regioninfo's seems good. Sidelining old hbase:meta");
1438 Path backupDir = sidelineOldMeta();
1439
1440 LOG.info("Creating new hbase:meta");
1441 HRegion meta = createNewMeta();
1442
1443
1444 List<Put> puts = generatePuts(tablesInfo);
1445 if (puts == null) {
1446 LOG.fatal("Problem encountered when creating new hbase:meta entries. " +
1447 "You may need to restore the previously sidelined hbase:meta");
1448 return false;
1449 }
1450 meta.batchMutate(puts.toArray(new Put[puts.size()]));
1451 HRegion.closeHRegion(meta);
1452 LOG.info("Success! hbase:meta table rebuilt.");
1453 LOG.info("Old hbase:meta is moved into " + backupDir);
1454 return true;
1455 }
1456
1457
1458
1459
1460 private void logParallelMerge() {
1461 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1462 LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1463 " false to run serially.");
1464 } else {
1465 LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to" +
1466 " true to run in parallel.");
1467 }
1468 }
1469
1470 private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1471 boolean fixOverlaps) throws IOException {
1472 LOG.info("Checking HBase region split map from HDFS data...");
1473 logParallelMerge();
1474 for (TableInfo tInfo : tablesInfo.values()) {
1475 TableIntegrityErrorHandler handler;
1476 if (fixHoles || fixOverlaps) {
1477 handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1478 fixHoles, fixOverlaps);
1479 } else {
1480 handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1481 }
1482 if (!tInfo.checkRegionChain(handler)) {
1483
1484 errors.report("Found inconsistency in table " + tInfo.getName());
1485 }
1486 }
1487 return tablesInfo;
1488 }
1489
1490 private Path getSidelineDir() throws IOException {
1491 if (sidelineDir == null) {
1492 Path hbaseDir = FSUtils.getRootDir(getConf());
1493 Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1494 sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1495 + startMillis);
1496 }
1497 return sidelineDir;
1498 }
1499
1500
1501
1502
1503 Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1504 return sidelineRegionDir(fs, null, hi);
1505 }
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515 Path sidelineRegionDir(FileSystem fs,
1516 String parentDir, HbckInfo hi) throws IOException {
1517 TableName tableName = hi.getTableName();
1518 Path regionDir = hi.getHdfsRegionDir();
1519
1520 if (!fs.exists(regionDir)) {
1521 LOG.warn("No previous " + regionDir + " exists. Continuing.");
1522 return null;
1523 }
1524
1525 Path rootDir = getSidelineDir();
1526 if (parentDir != null) {
1527 rootDir = new Path(rootDir, parentDir);
1528 }
1529 Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1530 Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1531 fs.mkdirs(sidelineRegionDir);
1532 boolean success = false;
1533 FileStatus[] cfs = fs.listStatus(regionDir);
1534 if (cfs == null) {
1535 LOG.info("Region dir is empty: " + regionDir);
1536 } else {
1537 for (FileStatus cf : cfs) {
1538 Path src = cf.getPath();
1539 Path dst = new Path(sidelineRegionDir, src.getName());
1540 if (fs.isFile(src)) {
1541
1542 success = fs.rename(src, dst);
1543 if (!success) {
1544 String msg = "Unable to rename file " + src + " to " + dst;
1545 LOG.error(msg);
1546 throw new IOException(msg);
1547 }
1548 continue;
1549 }
1550
1551
1552 fs.mkdirs(dst);
1553
1554 LOG.info("Sidelining files from " + src + " into containing region " + dst);
1555
1556
1557
1558
1559 FileStatus[] hfiles = fs.listStatus(src);
1560 if (hfiles != null && hfiles.length > 0) {
1561 for (FileStatus hfile : hfiles) {
1562 success = fs.rename(hfile.getPath(), dst);
1563 if (!success) {
1564 String msg = "Unable to rename file " + src + " to " + dst;
1565 LOG.error(msg);
1566 throw new IOException(msg);
1567 }
1568 }
1569 }
1570 LOG.debug("Sideline directory contents:");
1571 debugLsr(sidelineRegionDir);
1572 }
1573 }
1574
1575 LOG.info("Removing old region dir: " + regionDir);
1576 success = fs.delete(regionDir, true);
1577 if (!success) {
1578 String msg = "Unable to delete dir " + regionDir;
1579 LOG.error(msg);
1580 throw new IOException(msg);
1581 }
1582 return sidelineRegionDir;
1583 }
1584
1585
1586
1587
1588 void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1589 Path backupHbaseDir) throws IOException {
1590 Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1591 if (fs.exists(tableDir)) {
1592 Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1593 fs.mkdirs(backupTableDir.getParent());
1594 boolean success = fs.rename(tableDir, backupTableDir);
1595 if (!success) {
1596 throw new IOException("Failed to move " + tableName + " from "
1597 + tableDir + " to " + backupTableDir);
1598 }
1599 } else {
1600 LOG.info("No previous " + tableName + " exists. Continuing.");
1601 }
1602 }
1603
1604
1605
1606
1607 Path sidelineOldMeta() throws IOException {
1608
1609 Path hbaseDir = FSUtils.getRootDir(getConf());
1610 FileSystem fs = hbaseDir.getFileSystem(getConf());
1611 Path backupDir = getSidelineDir();
1612 fs.mkdirs(backupDir);
1613
1614 try {
1615 sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1616 } catch (IOException e) {
1617 LOG.fatal("... failed to sideline meta. Currently in inconsistent state. To restore "
1618 + "try to rename hbase:meta in " + backupDir.getName() + " to "
1619 + hbaseDir.getName() + ".", e);
1620 throw e;
1621 }
1622 return backupDir;
1623 }
1624
1625
1626
1627
1628
1629
1630 private void loadDisabledTables()
1631 throws ZooKeeperConnectionException, IOException {
1632 HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1633 @Override
1634 public Void connect(HConnection connection) throws IOException {
1635 ZooKeeperWatcher zkw = createZooKeeperWatcher();
1636 try {
1637 for (TableName tableName :
1638 ZKTableStateClientSideReader.getDisabledOrDisablingTables(zkw)) {
1639 disabledTables.add(tableName);
1640 }
1641 } catch (KeeperException ke) {
1642 throw new IOException(ke);
1643 } catch (InterruptedException e) {
1644 throw new InterruptedIOException();
1645 } finally {
1646 zkw.close();
1647 }
1648 return null;
1649 }
1650 });
1651 }
1652
1653
1654
1655
1656 private boolean isTableDisabled(HRegionInfo regionInfo) {
1657 return disabledTables.contains(regionInfo.getTable());
1658 }
1659
1660
1661
1662
1663
1664 public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1665 Path rootDir = FSUtils.getRootDir(getConf());
1666 FileSystem fs = rootDir.getFileSystem(getConf());
1667
1668
1669 List<FileStatus> tableDirs = Lists.newArrayList();
1670
1671 boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1672
1673 List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1674 for (Path path : paths) {
1675 TableName tableName = FSUtils.getTableName(path);
1676 if ((!checkMetaOnly &&
1677 isTableIncluded(tableName)) ||
1678 tableName.equals(TableName.META_TABLE_NAME)) {
1679 tableDirs.add(fs.getFileStatus(path));
1680 }
1681 }
1682
1683
1684 if (!foundVersionFile) {
1685 errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1686 "Version file does not exist in root dir " + rootDir);
1687 if (shouldFixVersionFile()) {
1688 LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1689 + " file.");
1690 setShouldRerun();
1691 FSUtils.setVersion(fs, rootDir, getConf().getInt(
1692 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1693 HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1694 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1695 }
1696 }
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723 private boolean recordMetaRegion() throws IOException {
1724 RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1725 HConstants.EMPTY_START_ROW, false, false);
1726 if (rl == null) {
1727 errors.reportError(ERROR_CODE.NULL_META_REGION,
1728 "META region or some of its attributes are null.");
1729 return false;
1730 }
1731 for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1732
1733 if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1734 metaLocation.getHostname() == null) {
1735 errors.reportError(ERROR_CODE.NULL_META_REGION,
1736 "META region or some of its attributes are null.");
1737 return false;
1738 }
1739 ServerName sn = metaLocation.getServerName();
1740 MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime());
1741 HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1742 if (hbckInfo == null) {
1743 regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1744 } else {
1745 hbckInfo.metaEntry = m;
1746 }
1747 }
1748 return true;
1749 }
1750
1751 private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1752 return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1753 @Override
1754 public void abort(String why, Throwable e) {
1755 LOG.error(why, e);
1756 System.exit(1);
1757 }
1758
1759 @Override
1760 public boolean isAborted() {
1761 return false;
1762 }
1763
1764 });
1765 }
1766
1767 private ServerName getMetaRegionServerName(int replicaId)
1768 throws IOException, KeeperException {
1769 ZooKeeperWatcher zkw = createZooKeeperWatcher();
1770 ServerName sn = null;
1771 try {
1772 sn = new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1773 } finally {
1774 zkw.close();
1775 }
1776 return sn;
1777 }
1778
1779
1780
1781
1782
1783
1784 void processRegionServers(Collection<ServerName> regionServerList)
1785 throws IOException, InterruptedException {
1786
1787 List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1788 List<Future<Void>> workFutures;
1789
1790
1791 for (ServerName rsinfo: regionServerList) {
1792 workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1793 }
1794
1795 workFutures = executor.invokeAll(workItems);
1796
1797 for(int i=0; i<workFutures.size(); i++) {
1798 WorkItemRegion item = workItems.get(i);
1799 Future<Void> f = workFutures.get(i);
1800 try {
1801 f.get();
1802 } catch(ExecutionException e) {
1803 LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1804 e.getCause());
1805 }
1806 }
1807 }
1808
1809
1810
1811
1812 private void checkAndFixConsistency()
1813 throws IOException, KeeperException, InterruptedException {
1814
1815
1816 List<CheckRegionConsistencyWorkItem> workItems =
1817 new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1818 for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1819 if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1820 workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1821 }
1822 }
1823 checkRegionConsistencyConcurrently(workItems);
1824
1825 boolean prevHdfsCheck = shouldCheckHdfs();
1826 setCheckHdfs(false);
1827
1828
1829 List<CheckRegionConsistencyWorkItem> replicaWorkItems =
1830 new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1831 for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1832 if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1833 replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1834 }
1835 }
1836 checkRegionConsistencyConcurrently(replicaWorkItems);
1837 setCheckHdfs(prevHdfsCheck);
1838
1839
1840
1841
1842
1843 int terminateThreshold = getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1844 int numOfSkippedRegions = skippedRegions.size();
1845 if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1846 throw new IOException(numOfSkippedRegions
1847 + " region(s) could not be checked or repaired. See logs for detail.");
1848 }
1849 }
1850
1851
1852
1853
1854 private void checkRegionConsistencyConcurrently(
1855 final List<CheckRegionConsistencyWorkItem> workItems)
1856 throws IOException, KeeperException, InterruptedException {
1857 if (workItems.isEmpty()) {
1858 return;
1859 }
1860
1861 List<Future<Void>> workFutures = executor.invokeAll(workItems);
1862 for(Future<Void> f: workFutures) {
1863 try {
1864 f.get();
1865 } catch(ExecutionException e1) {
1866 LOG.warn("Could not check region consistency " , e1.getCause());
1867 if (e1.getCause() instanceof IOException) {
1868 throw (IOException)e1.getCause();
1869 } else if (e1.getCause() instanceof KeeperException) {
1870 throw (KeeperException)e1.getCause();
1871 } else if (e1.getCause() instanceof InterruptedException) {
1872 throw (InterruptedException)e1.getCause();
1873 } else {
1874 throw new IOException(e1.getCause());
1875 }
1876 }
1877 }
1878 }
1879
1880 class CheckRegionConsistencyWorkItem implements Callable<Void> {
1881 private final String key;
1882 private final HbckInfo hbi;
1883
1884 CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) {
1885 this.key = key;
1886 this.hbi = hbi;
1887 }
1888
1889 @Override
1890 public synchronized Void call() throws Exception {
1891 try {
1892 checkRegionConsistency(key, hbi);
1893 } catch (Exception e) {
1894
1895
1896 LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1897 + "'.", e);
1898 if (hbi.getHdfsHRI().isMetaRegion()) {
1899 throw e;
1900 }
1901 LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1902 addSkippedRegion(hbi);
1903 }
1904 return null;
1905 }
1906 }
1907
1908 private void addSkippedRegion(final HbckInfo hbi) {
1909 Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1910 if (skippedRegionNames == null) {
1911 skippedRegionNames = new HashSet<String>();
1912 }
1913 skippedRegionNames.add(hbi.getRegionNameAsString());
1914 skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1915 }
1916
1917 private void preCheckPermission() throws IOException, AccessDeniedException {
1918 if (shouldIgnorePreCheckPermission()) {
1919 return;
1920 }
1921
1922 Path hbaseDir = FSUtils.getRootDir(getConf());
1923 FileSystem fs = hbaseDir.getFileSystem(getConf());
1924 UserProvider userProvider = UserProvider.instantiate(getConf());
1925 UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1926 FileStatus[] files = fs.listStatus(hbaseDir);
1927 for (FileStatus file : files) {
1928 try {
1929 FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1930 } catch (AccessDeniedException ace) {
1931 LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1932 errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1933 + " does not have write perms to " + file.getPath()
1934 + ". Please rerun hbck as hdfs user " + file.getOwner());
1935 throw ace;
1936 }
1937 }
1938 }
1939
1940
1941
1942
1943 private void deleteMetaRegion(HbckInfo hi) throws IOException {
1944 deleteMetaRegion(hi.metaEntry.getRegionName());
1945 }
1946
1947
1948
1949
1950 private void deleteMetaRegion(byte[] metaKey) throws IOException {
1951 Delete d = new Delete(metaKey);
1952 meta.delete(d);
1953 LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1954 }
1955
1956
1957
1958
1959 private void resetSplitParent(HbckInfo hi) throws IOException {
1960 RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1961 Delete d = new Delete(hi.metaEntry.getRegionName());
1962 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1963 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1964 mutations.add(d);
1965
1966 HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1967 hri.setOffline(false);
1968 hri.setSplit(false);
1969 Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1970 mutations.add(p);
1971
1972 meta.mutateRow(mutations);
1973 LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1974 }
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984 private void offline(byte[] regionName) throws IOException {
1985 String regionString = Bytes.toStringBinary(regionName);
1986 if (!rsSupportsOffline) {
1987 LOG.warn("Using unassign region " + regionString
1988 + " instead of using offline method, you should"
1989 + " restart HMaster after these repairs");
1990 admin.unassign(regionName, true);
1991 return;
1992 }
1993
1994
1995 try {
1996 LOG.info("Offlining region " + regionString);
1997 admin.offline(regionName);
1998 } catch (IOException ioe) {
1999 String notFoundMsg = "java.lang.NoSuchMethodException: " +
2000 "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2001 if (ioe.getMessage().contains(notFoundMsg)) {
2002 LOG.warn("Using unassign region " + regionString
2003 + " instead of using offline method, you should"
2004 + " restart HMaster after these repairs");
2005 rsSupportsOffline = false;
2006 admin.unassign(regionName, true);
2007 return;
2008 }
2009 throw ioe;
2010 }
2011 }
2012
2013 private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
2014 undeployRegionsForHbi(hi);
2015
2016 if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2017 return;
2018 }
2019 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2020 for (int i = 1; i < numReplicas; i++) {
2021 if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2022 HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2023 hi.getPrimaryHRIForDeployedReplica(), i);
2024 HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2025 if (h != null) {
2026 undeployRegionsForHbi(h);
2027
2028
2029 h.setSkipChecks(true);
2030 }
2031 }
2032 }
2033
2034 private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
2035 for (OnlineEntry rse : hi.deployedEntries) {
2036 LOG.debug("Undeploy region " + rse.hri + " from " + rse.hsa);
2037 try {
2038 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
2039 offline(rse.hri.getRegionName());
2040 } catch (IOException ioe) {
2041 LOG.warn("Got exception when attempting to offline region "
2042 + Bytes.toString(rse.hri.getRegionName()), ioe);
2043 }
2044 }
2045 }
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059 private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
2060 if (hi.metaEntry == null && hi.hdfsEntry == null) {
2061 undeployRegions(hi);
2062 return;
2063 }
2064
2065
2066 Get get = new Get(hi.getRegionName());
2067 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2068 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2069 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2070
2071 if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2072 int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2073 for (int i = 0; i < numReplicas; i++) {
2074 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2075 get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2076 }
2077 }
2078 Result r = meta.get(get);
2079 RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2080 if (rl == null) {
2081 LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2082 " since meta does not have handle to reach it");
2083 return;
2084 }
2085 for (HRegionLocation h : rl.getRegionLocations()) {
2086 ServerName serverName = h.getServerName();
2087 if (serverName == null) {
2088 errors.reportError("Unable to close region "
2089 + hi.getRegionNameAsString() + " because meta does not "
2090 + "have handle to reach it.");
2091 continue;
2092 }
2093 HRegionInfo hri = h.getRegionInfo();
2094 if (hri == null) {
2095 LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2096 + " because hbase:meta had invalid or missing "
2097 + HConstants.CATALOG_FAMILY_STR + ":"
2098 + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2099 + " qualifier value.");
2100 continue;
2101 }
2102
2103 HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2104 }
2105 }
2106
2107 private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
2108 KeeperException, InterruptedException {
2109
2110 if (shouldFixAssignments()) {
2111 errors.print(msg);
2112 undeployRegions(hbi);
2113 setShouldRerun();
2114 HRegionInfo hri = hbi.getHdfsHRI();
2115 if (hri == null) {
2116 hri = hbi.metaEntry;
2117 }
2118 HBaseFsckRepair.fixUnassigned(admin, hri);
2119 HBaseFsckRepair.waitUntilAssigned(admin, hri);
2120
2121
2122 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
2123 int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2124 for (int i = 1; i < replicationCount; i++) {
2125 hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2126 HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2127 if (h != null) {
2128 undeployRegions(h);
2129
2130
2131 h.setSkipChecks(true);
2132 }
2133 HBaseFsckRepair.fixUnassigned(admin, hri);
2134 HBaseFsckRepair.waitUntilAssigned(admin, hri);
2135 }
2136
2137 }
2138 }
2139
2140
2141
2142
2143 private void checkRegionConsistency(final String key, final HbckInfo hbi)
2144 throws IOException, KeeperException, InterruptedException {
2145
2146 if (hbi.isSkipChecks()) return;
2147 String descriptiveName = hbi.toString();
2148 boolean inMeta = hbi.metaEntry != null;
2149
2150 boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2151 boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2152 boolean isDeployed = !hbi.deployedOn.isEmpty();
2153 boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2154 boolean deploymentMatchesMeta =
2155 hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2156 hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2157 boolean splitParent =
2158 (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2159 boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
2160 boolean recentlyModified = inHdfs &&
2161 hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2162
2163
2164 if (hbi.containsOnlyHdfsEdits()) {
2165 return;
2166 }
2167 if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2168 return;
2169 } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2170 LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2171 "tabled that is not deployed");
2172 return;
2173 } else if (recentlyModified) {
2174 LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2175 return;
2176 }
2177
2178 else if (!inMeta && !inHdfs && !isDeployed) {
2179
2180 assert false : "Entry for region with no data";
2181 } else if (!inMeta && !inHdfs && isDeployed) {
2182 errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2183 + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2184 "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2185 if (shouldFixAssignments()) {
2186 undeployRegions(hbi);
2187 }
2188
2189 } else if (!inMeta && inHdfs && !isDeployed) {
2190 if (hbi.isMerged()) {
2191
2192
2193 hbi.setSkipChecks(true);
2194 LOG.info("Region " + descriptiveName
2195 + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2196 return;
2197 }
2198 errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2199 + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2200 "or deployed on any region server");
2201
2202 if (shouldFixMeta()) {
2203 if (!hbi.isHdfsRegioninfoPresent()) {
2204 LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2205 + " in table integrity repair phase if -fixHdfsOrphans was" +
2206 " used.");
2207 return;
2208 }
2209
2210 HRegionInfo hri = hbi.getHdfsHRI();
2211 TableInfo tableInfo = tablesInfo.get(hri.getTable());
2212
2213 for (HRegionInfo region : tableInfo.getRegionsFromMeta()) {
2214 if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2215 && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2216 hri.getEndKey()) >= 0)
2217 && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2218 if(region.isSplit() || region.isOffline()) continue;
2219 Path regionDir = hbi.getHdfsRegionDir();
2220 FileSystem fs = regionDir.getFileSystem(getConf());
2221 List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2222 for (Path familyDir : familyDirs) {
2223 List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2224 for (Path referenceFilePath : referenceFilePaths) {
2225 Path parentRegionDir =
2226 StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2227 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2228 LOG.warn(hri + " start and stop keys are in the range of " + region
2229 + ". The region might not be cleaned up from hdfs when region " + region
2230 + " split failed. Hence deleting from hdfs.");
2231 HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2232 regionDir.getParent(), hri);
2233 return;
2234 }
2235 }
2236 }
2237 }
2238 }
2239
2240 LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2241 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2242 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2243 admin.getClusterStatus().getServers(), numReplicas);
2244
2245 tryAssignmentRepair(hbi, "Trying to reassign region...");
2246 }
2247
2248 } else if (!inMeta && inHdfs && isDeployed) {
2249 errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2250 + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2251 debugLsr(hbi.getHdfsRegionDir());
2252 if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2253
2254
2255
2256
2257 if (shouldFixAssignments()) {
2258 undeployRegionsForHbi(hbi);
2259 }
2260 }
2261 if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2262 if (!hbi.isHdfsRegioninfoPresent()) {
2263 LOG.error("This should have been repaired in table integrity repair phase");
2264 return;
2265 }
2266
2267 LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2268 int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2269 HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2270 admin.getClusterStatus().getServers(), numReplicas);
2271 tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2272 }
2273
2274
2275 } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2276
2277
2278 if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2279
2280 HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2281 HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2282 if (infoA != null && infoB != null) {
2283
2284 hbi.setSkipChecks(true);
2285 return;
2286 }
2287 }
2288 errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2289 + descriptiveName + " is a split parent in META, in HDFS, "
2290 + "and not deployed on any region server. This could be transient.");
2291 if (shouldFixSplitParents()) {
2292 setShouldRerun();
2293 resetSplitParent(hbi);
2294 }
2295 } else if (inMeta && !inHdfs && !isDeployed) {
2296 errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2297 + descriptiveName + " found in META, but not in HDFS "
2298 + "or deployed on any region server.");
2299 if (shouldFixMeta()) {
2300 deleteMetaRegion(hbi);
2301 }
2302 } else if (inMeta && !inHdfs && isDeployed) {
2303 errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2304 + " found in META, but not in HDFS, " +
2305 "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2306
2307
2308
2309 if (shouldFixAssignments()) {
2310 errors.print("Trying to fix unassigned region...");
2311 undeployRegions(hbi);
2312 }
2313 if (shouldFixMeta()) {
2314
2315 deleteMetaRegion(hbi);
2316 }
2317 } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2318 errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2319 + " not deployed on any region server.");
2320 tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2321 } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2322 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2323 "Region " + descriptiveName + " should not be deployed according " +
2324 "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2325 if (shouldFixAssignments()) {
2326 errors.print("Trying to close the region " + descriptiveName);
2327 setShouldRerun();
2328 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2329 }
2330 } else if (inMeta && inHdfs && isMultiplyDeployed) {
2331 errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2332 + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2333 + " but is multiply assigned to region servers " +
2334 Joiner.on(", ").join(hbi.deployedOn));
2335
2336 if (shouldFixAssignments()) {
2337 errors.print("Trying to fix assignment error...");
2338 setShouldRerun();
2339 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2340 }
2341 } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2342 errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2343 + descriptiveName + " listed in hbase:meta on region server " +
2344 hbi.metaEntry.regionServer + " but found on region server " +
2345 hbi.deployedOn.get(0));
2346
2347 if (shouldFixAssignments()) {
2348 errors.print("Trying to fix assignment error...");
2349 setShouldRerun();
2350 HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2351 HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2352 }
2353 } else {
2354 errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2355 " is in an unforeseen state:" +
2356 " inMeta=" + inMeta +
2357 " inHdfs=" + inHdfs +
2358 " isDeployed=" + isDeployed +
2359 " isMultiplyDeployed=" + isMultiplyDeployed +
2360 " deploymentMatchesMeta=" + deploymentMatchesMeta +
2361 " shouldBeDeployed=" + shouldBeDeployed);
2362 }
2363 }
2364
2365
2366
2367
2368
2369
2370
2371 SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2372 tablesInfo = new TreeMap<TableName,TableInfo> ();
2373 LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2374 for (HbckInfo hbi : regionInfoMap.values()) {
2375
2376 if (hbi.metaEntry == null) {
2377
2378 Path p = hbi.getHdfsRegionDir();
2379 if (p == null) {
2380 errors.report("No regioninfo in Meta or HDFS. " + hbi);
2381 }
2382
2383
2384 continue;
2385 }
2386 if (hbi.metaEntry.regionServer == null) {
2387 errors.detail("Skipping region because no region server: " + hbi);
2388 continue;
2389 }
2390 if (hbi.metaEntry.isOffline()) {
2391 errors.detail("Skipping region because it is offline: " + hbi);
2392 continue;
2393 }
2394 if (hbi.containsOnlyHdfsEdits()) {
2395 errors.detail("Skipping region because it only contains edits" + hbi);
2396 continue;
2397 }
2398
2399
2400
2401
2402
2403
2404 if (hbi.deployedOn.size() == 0) continue;
2405
2406
2407 TableName tableName = hbi.metaEntry.getTable();
2408 TableInfo modTInfo = tablesInfo.get(tableName);
2409 if (modTInfo == null) {
2410 modTInfo = new TableInfo(tableName);
2411 }
2412 for (ServerName server : hbi.deployedOn) {
2413 modTInfo.addServer(server);
2414 }
2415
2416 if (!hbi.isSkipChecks()) {
2417 modTInfo.addRegionInfo(hbi);
2418 }
2419
2420 tablesInfo.put(tableName, modTInfo);
2421 }
2422
2423 loadTableInfosForTablesWithNoRegion();
2424
2425 logParallelMerge();
2426 for (TableInfo tInfo : tablesInfo.values()) {
2427 TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2428 if (!tInfo.checkRegionChain(handler)) {
2429 errors.report("Found inconsistency in table " + tInfo.getName());
2430 }
2431 }
2432 return tablesInfo;
2433 }
2434
2435
2436
2437
2438 private void loadTableInfosForTablesWithNoRegion() throws IOException {
2439 Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2440 for (HTableDescriptor htd : allTables.values()) {
2441 if (checkMetaOnly && !htd.isMetaTable()) {
2442 continue;
2443 }
2444
2445 TableName tableName = htd.getTableName();
2446 if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2447 TableInfo tableInfo = new TableInfo(tableName);
2448 tableInfo.htds.add(htd);
2449 tablesInfo.put(htd.getTableName(), tableInfo);
2450 }
2451 }
2452 }
2453
2454
2455
2456
2457
2458 public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2459 int fileMoves = 0;
2460 String thread = Thread.currentThread().getName();
2461 LOG.debug("[" + thread + "] Contained region dir after close and pause");
2462 debugLsr(contained.getHdfsRegionDir());
2463
2464
2465 FileSystem fs = targetRegionDir.getFileSystem(getConf());
2466 FileStatus[] dirs = null;
2467 try {
2468 dirs = fs.listStatus(contained.getHdfsRegionDir());
2469 } catch (FileNotFoundException fnfe) {
2470
2471
2472 if (!fs.exists(contained.getHdfsRegionDir())) {
2473 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2474 + " is missing. Assuming already sidelined or moved.");
2475 } else {
2476 sidelineRegionDir(fs, contained);
2477 }
2478 return fileMoves;
2479 }
2480
2481 if (dirs == null) {
2482 if (!fs.exists(contained.getHdfsRegionDir())) {
2483 LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2484 + " already sidelined.");
2485 } else {
2486 sidelineRegionDir(fs, contained);
2487 }
2488 return fileMoves;
2489 }
2490
2491 for (FileStatus cf : dirs) {
2492 Path src = cf.getPath();
2493 Path dst = new Path(targetRegionDir, src.getName());
2494
2495 if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2496
2497 continue;
2498 }
2499
2500 if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2501
2502 continue;
2503 }
2504
2505 LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2506
2507
2508
2509
2510 for (FileStatus hfile : fs.listStatus(src)) {
2511 boolean success = fs.rename(hfile.getPath(), dst);
2512 if (success) {
2513 fileMoves++;
2514 }
2515 }
2516 LOG.debug("[" + thread + "] Sideline directory contents:");
2517 debugLsr(targetRegionDir);
2518 }
2519
2520
2521 sidelineRegionDir(fs, contained);
2522 LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2523 getSidelineDir());
2524 debugLsr(contained.getHdfsRegionDir());
2525
2526 return fileMoves;
2527 }
2528
2529
2530 static class WorkItemOverlapMerge implements Callable<Void> {
2531 private TableIntegrityErrorHandler handler;
2532 Collection<HbckInfo> overlapgroup;
2533
2534 WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2535 this.handler = handler;
2536 this.overlapgroup = overlapgroup;
2537 }
2538
2539 @Override
2540 public Void call() throws Exception {
2541 handler.handleOverlapGroup(overlapgroup);
2542 return null;
2543 }
2544 };
2545
2546
2547
2548
2549
2550 public class TableInfo {
2551 TableName tableName;
2552 TreeSet <ServerName> deployedOn;
2553
2554
2555 final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2556
2557
2558 final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2559
2560
2561 final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2562
2563
2564 final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2565
2566
2567 final Multimap<byte[], HbckInfo> overlapGroups =
2568 TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2569
2570
2571 private ImmutableList<HRegionInfo> regionsFromMeta = null;
2572
2573 TableInfo(TableName name) {
2574 this.tableName = name;
2575 deployedOn = new TreeSet <ServerName>();
2576 }
2577
2578
2579
2580
2581 private HTableDescriptor getHTD() {
2582 if (htds.size() == 1) {
2583 return (HTableDescriptor)htds.toArray()[0];
2584 } else {
2585 LOG.error("None/Multiple table descriptors found for table '"
2586 + tableName + "' regions: " + htds);
2587 }
2588 return null;
2589 }
2590
2591 public void addRegionInfo(HbckInfo hir) {
2592 if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2593
2594
2595 if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2596 return;
2597 }
2598
2599
2600 if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2601 errors.reportError(
2602 ERROR_CODE.REGION_CYCLE,
2603 String.format("The endkey for this region comes before the "
2604 + "startkey, startkey=%s, endkey=%s",
2605 Bytes.toStringBinary(hir.getStartKey()),
2606 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2607 backwards.add(hir);
2608 return;
2609 }
2610
2611
2612
2613 if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2614 }
2615
2616 public void addServer(ServerName server) {
2617 this.deployedOn.add(server);
2618 }
2619
2620 public TableName getName() {
2621 return tableName;
2622 }
2623
2624 public int getNumRegions() {
2625 return sc.getStarts().size() + backwards.size();
2626 }
2627
2628 public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() {
2629
2630 if (regionsFromMeta == null) {
2631 List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2632 for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) {
2633 if (tableName.equals(h.getTableName())) {
2634 if (h.metaEntry != null) {
2635 regions.add((HRegionInfo) h.metaEntry);
2636 }
2637 }
2638 }
2639 regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
2640 }
2641
2642 return regionsFromMeta;
2643 }
2644
2645
2646 private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2647 ErrorReporter errors;
2648
2649 IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2650 this.errors = errors;
2651 setTableInfo(ti);
2652 }
2653
2654 @Override
2655 public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2656 errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2657 "First region should start with an empty key. You need to "
2658 + " create a new region and regioninfo in HDFS to plug the hole.",
2659 getTableInfo(), hi);
2660 }
2661
2662 @Override
2663 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2664 errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2665 "Last region should end with an empty key. You need to "
2666 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2667 }
2668
2669 @Override
2670 public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2671 errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2672 "Region has the same start and end key.", getTableInfo(), hi);
2673 }
2674
2675 @Override
2676 public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2677 byte[] key = r1.getStartKey();
2678
2679 errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2680 "Multiple regions have the same startkey: "
2681 + Bytes.toStringBinary(key), getTableInfo(), r1);
2682 errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2683 "Multiple regions have the same startkey: "
2684 + Bytes.toStringBinary(key), getTableInfo(), r2);
2685 }
2686
2687 @Override
2688 public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2689 errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2690 "There is an overlap in the region chain.",
2691 getTableInfo(), hi1, hi2);
2692 }
2693
2694 @Override
2695 public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2696 errors.reportError(
2697 ERROR_CODE.HOLE_IN_REGION_CHAIN,
2698 "There is a hole in the region chain between "
2699 + Bytes.toStringBinary(holeStart) + " and "
2700 + Bytes.toStringBinary(holeStop)
2701 + ". You need to create a new .regioninfo and region "
2702 + "dir in hdfs to plug the hole.");
2703 }
2704 };
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718 private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2719 Configuration conf;
2720
2721 boolean fixOverlaps = true;
2722
2723 HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2724 boolean fixHoles, boolean fixOverlaps) {
2725 super(ti, errors);
2726 this.conf = conf;
2727 this.fixOverlaps = fixOverlaps;
2728
2729 }
2730
2731
2732
2733
2734
2735
2736 @Override
2737 public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2738 errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2739 "First region should start with an empty key. Creating a new " +
2740 "region and regioninfo in HDFS to plug the hole.",
2741 getTableInfo(), next);
2742 HTableDescriptor htd = getTableInfo().getHTD();
2743
2744 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2745 HConstants.EMPTY_START_ROW, next.getStartKey());
2746
2747
2748 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2749 LOG.info("Table region start key was not empty. Created new empty region: "
2750 + newRegion + " " +region);
2751 fixes++;
2752 }
2753
2754 @Override
2755 public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2756 errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2757 "Last region should end with an empty key. Creating a new "
2758 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2759 HTableDescriptor htd = getTableInfo().getHTD();
2760
2761 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2762 HConstants.EMPTY_START_ROW);
2763
2764 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2765 LOG.info("Table region end key was not empty. Created new empty region: " + newRegion
2766 + " " + region);
2767 fixes++;
2768 }
2769
2770
2771
2772
2773
2774 @Override
2775 public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2776 errors.reportError(
2777 ERROR_CODE.HOLE_IN_REGION_CHAIN,
2778 "There is a hole in the region chain between "
2779 + Bytes.toStringBinary(holeStartKey) + " and "
2780 + Bytes.toStringBinary(holeStopKey)
2781 + ". Creating a new regioninfo and region "
2782 + "dir in hdfs to plug the hole.");
2783 HTableDescriptor htd = getTableInfo().getHTD();
2784 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2785 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2786 LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2787 fixes++;
2788 }
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801 @Override
2802 public void handleOverlapGroup(Collection<HbckInfo> overlap)
2803 throws IOException {
2804 Preconditions.checkNotNull(overlap);
2805 Preconditions.checkArgument(overlap.size() >0);
2806
2807 if (!this.fixOverlaps) {
2808 LOG.warn("Not attempting to repair overlaps.");
2809 return;
2810 }
2811
2812 if (overlap.size() > maxMerge) {
2813 LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2814 "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2815 if (sidelineBigOverlaps) {
2816
2817 sidelineBigOverlaps(overlap);
2818 }
2819 return;
2820 }
2821
2822 mergeOverlaps(overlap);
2823 }
2824
2825 void mergeOverlaps(Collection<HbckInfo> overlap)
2826 throws IOException {
2827 String thread = Thread.currentThread().getName();
2828 LOG.info("== [" + thread + "] Merging regions into one region: "
2829 + Joiner.on(",").join(overlap));
2830
2831 Pair<byte[], byte[]> range = null;
2832 for (HbckInfo hi : overlap) {
2833 if (range == null) {
2834 range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2835 } else {
2836 if (RegionSplitCalculator.BYTES_COMPARATOR
2837 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2838 range.setFirst(hi.getStartKey());
2839 }
2840 if (RegionSplitCalculator.BYTES_COMPARATOR
2841 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2842 range.setSecond(hi.getEndKey());
2843 }
2844 }
2845
2846 LOG.debug("[" + thread + "] Closing region before moving data around: " + hi);
2847 LOG.debug("[" + thread + "] Contained region dir before close");
2848 debugLsr(hi.getHdfsRegionDir());
2849 try {
2850 LOG.info("[" + thread + "] Closing region: " + hi);
2851 closeRegion(hi);
2852 } catch (IOException ioe) {
2853 LOG.warn("[" + thread + "] Was unable to close region " + hi
2854 + ". Just continuing... ", ioe);
2855 } catch (InterruptedException e) {
2856 LOG.warn("[" + thread + "] Was unable to close region " + hi
2857 + ". Just continuing... ", e);
2858 }
2859
2860 try {
2861 LOG.info("[" + thread + "] Offlining region: " + hi);
2862 offline(hi.getRegionName());
2863 } catch (IOException ioe) {
2864 LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2865 + ". Just continuing... ", ioe);
2866 }
2867 }
2868
2869
2870 HTableDescriptor htd = getTableInfo().getHTD();
2871
2872 HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2873 range.getSecond());
2874 HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2875 LOG.info("[" + thread + "] Created new empty container region: " +
2876 newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2877 debugLsr(region.getRegionFileSystem().getRegionDir());
2878
2879
2880 boolean didFix= false;
2881 Path target = region.getRegionFileSystem().getRegionDir();
2882 for (HbckInfo contained : overlap) {
2883 LOG.info("[" + thread + "] Merging " + contained + " into " + target );
2884 int merges = mergeRegionDirs(target, contained);
2885 if (merges > 0) {
2886 didFix = true;
2887 }
2888 }
2889 if (didFix) {
2890 fixes++;
2891 }
2892 }
2893
2894
2895
2896
2897
2898
2899
2900
2901 void sidelineBigOverlaps(
2902 Collection<HbckInfo> bigOverlap) throws IOException {
2903 int overlapsToSideline = bigOverlap.size() - maxMerge;
2904 if (overlapsToSideline > maxOverlapsToSideline) {
2905 overlapsToSideline = maxOverlapsToSideline;
2906 }
2907 List<HbckInfo> regionsToSideline =
2908 RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2909 FileSystem fs = FileSystem.get(conf);
2910 for (HbckInfo regionToSideline: regionsToSideline) {
2911 try {
2912 LOG.info("Closing region: " + regionToSideline);
2913 closeRegion(regionToSideline);
2914 } catch (IOException ioe) {
2915 LOG.warn("Was unable to close region " + regionToSideline
2916 + ". Just continuing... ", ioe);
2917 } catch (InterruptedException e) {
2918 LOG.warn("Was unable to close region " + regionToSideline
2919 + ". Just continuing... ", e);
2920 }
2921
2922 try {
2923 LOG.info("Offlining region: " + regionToSideline);
2924 offline(regionToSideline.getRegionName());
2925 } catch (IOException ioe) {
2926 LOG.warn("Unable to offline region from master: " + regionToSideline
2927 + ". Just continuing... ", ioe);
2928 }
2929
2930 LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2931 Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2932 if (sidelineRegionDir != null) {
2933 sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2934 LOG.info("After sidelined big overlapped region: "
2935 + regionToSideline.getRegionNameAsString()
2936 + " to " + sidelineRegionDir.toString());
2937 fixes++;
2938 }
2939 }
2940 }
2941 }
2942
2943
2944
2945
2946
2947
2948
2949 public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2950
2951
2952
2953 if (disabledTables.contains(this.tableName)) {
2954 return true;
2955 }
2956 int originalErrorsCount = errors.getErrorList().size();
2957 Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2958 SortedSet<byte[]> splits = sc.getSplits();
2959
2960 byte[] prevKey = null;
2961 byte[] problemKey = null;
2962
2963 if (splits.size() == 0) {
2964
2965 handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2966 }
2967
2968 for (byte[] key : splits) {
2969 Collection<HbckInfo> ranges = regions.get(key);
2970 if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2971 for (HbckInfo rng : ranges) {
2972 handler.handleRegionStartKeyNotEmpty(rng);
2973 }
2974 }
2975
2976
2977 for (HbckInfo rng : ranges) {
2978
2979 byte[] endKey = rng.getEndKey();
2980 endKey = (endKey.length == 0) ? null : endKey;
2981 if (Bytes.equals(rng.getStartKey(),endKey)) {
2982 handler.handleDegenerateRegion(rng);
2983 }
2984 }
2985
2986 if (ranges.size() == 1) {
2987
2988 if (problemKey != null) {
2989 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2990 }
2991 problemKey = null;
2992 } else if (ranges.size() > 1) {
2993
2994
2995 if (problemKey == null) {
2996
2997 LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2998 problemKey = key;
2999 }
3000 overlapGroups.putAll(problemKey, ranges);
3001
3002
3003 ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
3004
3005 for (HbckInfo r1 : ranges) {
3006 if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3007 subRange.remove(r1);
3008 for (HbckInfo r2 : subRange) {
3009 if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3010 if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
3011 handler.handleDuplicateStartKeys(r1,r2);
3012 } else {
3013
3014 handler.handleOverlapInRegionChain(r1, r2);
3015 }
3016 }
3017 }
3018
3019 } else if (ranges.size() == 0) {
3020 if (problemKey != null) {
3021 LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3022 }
3023 problemKey = null;
3024
3025 byte[] holeStopKey = sc.getSplits().higher(key);
3026
3027 if (holeStopKey != null) {
3028
3029 handler.handleHoleInRegionChain(key, holeStopKey);
3030 }
3031 }
3032 prevKey = key;
3033 }
3034
3035
3036
3037 if (prevKey != null) {
3038 handler.handleRegionEndKeyNotEmpty(prevKey);
3039 }
3040
3041
3042 if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
3043 boolean ok = handleOverlapsParallel(handler, prevKey);
3044 if (!ok) {
3045 return false;
3046 }
3047 } else {
3048 for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3049 handler.handleOverlapGroup(overlap);
3050 }
3051 }
3052
3053 if (details) {
3054
3055 errors.print("---- Table '" + this.tableName
3056 + "': region split map");
3057 dump(splits, regions);
3058 errors.print("---- Table '" + this.tableName
3059 + "': overlap groups");
3060 dumpOverlapProblems(overlapGroups);
3061 errors.print("There are " + overlapGroups.keySet().size()
3062 + " overlap groups with " + overlapGroups.size()
3063 + " overlapping regions");
3064 }
3065 if (!sidelinedRegions.isEmpty()) {
3066 LOG.warn("Sidelined big overlapped regions, please bulk load them!");
3067 errors.print("---- Table '" + this.tableName
3068 + "': sidelined big overlapped regions");
3069 dumpSidelinedRegions(sidelinedRegions);
3070 }
3071 return errors.getErrorList().size() == originalErrorsCount;
3072 }
3073
3074 private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
3075 throws IOException {
3076
3077
3078 List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
3079 List<Future<Void>> rets;
3080 for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3081
3082 merges.add(new WorkItemOverlapMerge(overlap, handler));
3083 }
3084 try {
3085 rets = executor.invokeAll(merges);
3086 } catch (InterruptedException e) {
3087 LOG.error("Overlap merges were interrupted", e);
3088 return false;
3089 }
3090 for(int i=0; i<merges.size(); i++) {
3091 WorkItemOverlapMerge work = merges.get(i);
3092 Future<Void> f = rets.get(i);
3093 try {
3094 f.get();
3095 } catch(ExecutionException e) {
3096 LOG.warn("Failed to merge overlap group" + work, e.getCause());
3097 } catch (InterruptedException e) {
3098 LOG.error("Waiting for overlap merges was interrupted", e);
3099 return false;
3100 }
3101 }
3102 return true;
3103 }
3104
3105
3106
3107
3108
3109
3110
3111 void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
3112
3113 StringBuilder sb = new StringBuilder();
3114 for (byte[] k : splits) {
3115 sb.setLength(0);
3116 sb.append(Bytes.toStringBinary(k) + ":\t");
3117 for (HbckInfo r : regions.get(k)) {
3118 sb.append("[ "+ r.toString() + ", "
3119 + Bytes.toStringBinary(r.getEndKey())+ "]\t");
3120 }
3121 errors.print(sb.toString());
3122 }
3123 }
3124 }
3125
3126 public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
3127
3128
3129 for (byte[] k : regions.keySet()) {
3130 errors.print(Bytes.toStringBinary(k) + ":");
3131 for (HbckInfo r : regions.get(k)) {
3132 errors.print("[ " + r.toString() + ", "
3133 + Bytes.toStringBinary(r.getEndKey()) + "]");
3134 }
3135 errors.print("----");
3136 }
3137 }
3138
3139 public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
3140 for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
3141 TableName tableName = entry.getValue().getTableName();
3142 Path path = entry.getKey();
3143 errors.print("This sidelined region dir should be bulk loaded: "
3144 + path.toString());
3145 errors.print("Bulk load command looks like: "
3146 + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
3147 + path.toUri().getPath() + " "+ tableName);
3148 }
3149 }
3150
3151 public Multimap<byte[], HbckInfo> getOverlapGroups(
3152 TableName table) {
3153 TableInfo ti = tablesInfo.get(table);
3154 return ti.overlapGroups;
3155 }
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166 HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3167 List<TableName> tableNames = new ArrayList<TableName>();
3168 long now = EnvironmentEdgeManager.currentTime();
3169
3170 for (HbckInfo hbi : regionInfoMap.values()) {
3171 MetaEntry info = hbi.metaEntry;
3172
3173
3174
3175 if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3176 if (info.modTime + timelag < now) {
3177 tableNames.add(info.getTable());
3178 } else {
3179 numSkipped.incrementAndGet();
3180 }
3181 }
3182 }
3183 return getHTableDescriptors(tableNames);
3184 }
3185
3186 HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3187 HTableDescriptor[] htd = new HTableDescriptor[0];
3188 Admin admin = null;
3189 try {
3190 LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3191 admin = new HBaseAdmin(getConf());
3192 htd = admin.getTableDescriptorsByTableName(tableNames);
3193 } catch (IOException e) {
3194 LOG.debug("Exception getting table descriptors", e);
3195 } finally {
3196 if (admin != null) {
3197 try {
3198 admin.close();
3199 } catch (IOException e) {
3200 LOG.debug("Exception closing HBaseAdmin", e);
3201 }
3202 }
3203 }
3204 return htd;
3205 }
3206
3207
3208
3209
3210
3211
3212 private synchronized HbckInfo getOrCreateInfo(String name) {
3213 HbckInfo hbi = regionInfoMap.get(name);
3214 if (hbi == null) {
3215 hbi = new HbckInfo(null);
3216 regionInfoMap.put(name, hbi);
3217 }
3218 return hbi;
3219 }
3220
3221 private void checkAndFixTableLocks() throws IOException {
3222 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3223
3224 try {
3225 TableLockChecker checker = new TableLockChecker(zkw, errors);
3226 checker.checkTableLocks();
3227
3228 if (this.fixTableLocks) {
3229 checker.fixExpiredTableLocks();
3230 }
3231 } finally {
3232 zkw.close();
3233 }
3234 }
3235
3236 private void checkAndFixReplication() throws IOException {
3237 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3238 try {
3239 ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);
3240 checker.checkUnDeletedQueues();
3241
3242 if (checker.hasUnDeletedQueues() && this.fixReplication) {
3243 checker.fixUnDeletedQueues();
3244 setShouldRerun();
3245 }
3246 } finally {
3247 zkw.close();
3248 }
3249 }
3250
3251
3252
3253
3254
3255
3256
3257 private void checkAndFixOrphanedTableZNodes()
3258 throws IOException, KeeperException, InterruptedException {
3259 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3260
3261 try {
3262 Set<TableName> enablingTables = ZKTableStateClientSideReader.getEnablingTables(zkw);
3263 String msg;
3264 TableInfo tableInfo;
3265
3266 for (TableName tableName : enablingTables) {
3267
3268 tableInfo = tablesInfo.get(tableName);
3269 if (tableInfo != null) {
3270
3271 continue;
3272 }
3273
3274 msg = "Table " + tableName + " not found in hbase:meta. Orphaned table ZNode found.";
3275 LOG.warn(msg);
3276 orphanedTableZNodes.add(tableName);
3277 errors.reportError(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY, msg);
3278 }
3279
3280 if (orphanedTableZNodes.size() > 0 && this.fixTableZNodes) {
3281 ZKTableStateManager zkTableStateMgr = new ZKTableStateManager(zkw);
3282
3283 for (TableName tableName : orphanedTableZNodes) {
3284 try {
3285
3286
3287
3288
3289 zkTableStateMgr.setTableState(tableName, ZooKeeperProtos.Table.State.DISABLED);
3290 } catch (CoordinatedStateException e) {
3291
3292 LOG.error(
3293 "Got a CoordinatedStateException while fixing the ENABLING table znode " + tableName,
3294 e);
3295 }
3296 }
3297 }
3298 } finally {
3299 zkw.close();
3300 }
3301 }
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312 boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3313 Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3314 for (HbckInfo value : regionInfoMap.values()) {
3315 if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3316 metaRegions.put(value.getReplicaId(), value);
3317 }
3318 }
3319 int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3320 .getRegionReplication();
3321 boolean noProblem = true;
3322
3323
3324 for (int i = 0; i < metaReplication; i++) {
3325 HbckInfo metaHbckInfo = metaRegions.remove(i);
3326 List<ServerName> servers = new ArrayList<ServerName>();
3327 if (metaHbckInfo != null) {
3328 servers = metaHbckInfo.deployedOn;
3329 }
3330 if (servers.size() != 1) {
3331 noProblem = false;
3332 if (servers.size() == 0) {
3333 assignMetaReplica(i);
3334 } else if (servers.size() > 1) {
3335 errors
3336 .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3337 metaHbckInfo.getReplicaId() + " is found on more than one region.");
3338 if (shouldFixAssignments()) {
3339 errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3340 metaHbckInfo.getReplicaId() +"..");
3341 setShouldRerun();
3342
3343 HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3344 }
3345 }
3346 }
3347 }
3348
3349 for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3350 noProblem = false;
3351 errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3352 "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3353 ", deployed " + metaRegions.size());
3354 if (shouldFixAssignments()) {
3355 errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3356 " of hbase:meta..");
3357 setShouldRerun();
3358 unassignMetaReplica(entry.getValue());
3359 }
3360 }
3361
3362
3363 return noProblem;
3364 }
3365
3366 private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3367 KeeperException {
3368 undeployRegions(hi);
3369 ZooKeeperWatcher zkw = createZooKeeperWatcher();
3370 try {
3371 ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3372 } finally {
3373 zkw.close();
3374 }
3375 }
3376
3377 private void assignMetaReplica(int replicaId)
3378 throws IOException, KeeperException, InterruptedException {
3379 errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3380 replicaId +" is not found on any region.");
3381 if (shouldFixAssignments()) {
3382 errors.print("Trying to fix a problem with hbase:meta..");
3383 setShouldRerun();
3384
3385 HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3386 HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3387 HBaseFsckRepair.fixUnassigned(admin, h);
3388 HBaseFsckRepair.waitUntilAssigned(admin, h);
3389 }
3390 }
3391
3392
3393
3394
3395
3396 boolean loadMetaEntries() throws IOException {
3397 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
3398 int countRecord = 1;
3399
3400
3401 final Comparator<Cell> comp = new Comparator<Cell>() {
3402 @Override
3403 public int compare(Cell k1, Cell k2) {
3404 return (int)(k1.getTimestamp() - k2.getTimestamp());
3405 }
3406 };
3407
3408 @Override
3409 public boolean processRow(Result result) throws IOException {
3410 try {
3411
3412
3413 long ts = Collections.max(result.listCells(), comp).getTimestamp();
3414 RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3415 if (rl == null) {
3416 emptyRegionInfoQualifiers.add(result);
3417 errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3418 "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3419 return true;
3420 }
3421 ServerName sn = null;
3422 if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3423 rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3424 emptyRegionInfoQualifiers.add(result);
3425 errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3426 "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3427 return true;
3428 }
3429 HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3430 if (!(isTableIncluded(hri.getTable())
3431 || hri.isMetaRegion())) {
3432 return true;
3433 }
3434 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3435 for (HRegionLocation h : rl.getRegionLocations()) {
3436 if (h == null || h.getRegionInfo() == null) {
3437 continue;
3438 }
3439 sn = h.getServerName();
3440 hri = h.getRegionInfo();
3441
3442 MetaEntry m = null;
3443 if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3444 m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3445 } else {
3446 m = new MetaEntry(hri, sn, ts, null, null);
3447 }
3448 HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3449 if (previous == null) {
3450 regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3451 } else if (previous.metaEntry == null) {
3452 previous.metaEntry = m;
3453 } else {
3454 throw new IOException("Two entries in hbase:meta are same " + previous);
3455 }
3456 }
3457 PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3458 for (HRegionInfo mergeRegion : new HRegionInfo[] {
3459 mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3460 if (mergeRegion != null) {
3461
3462 HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3463 hbInfo.setMerged(true);
3464 }
3465 }
3466
3467
3468 if (countRecord % 100 == 0) {
3469 errors.progress();
3470 }
3471 countRecord++;
3472 return true;
3473 } catch (RuntimeException e) {
3474 LOG.error("Result=" + result);
3475 throw e;
3476 }
3477 }
3478 };
3479 if (!checkMetaOnly) {
3480
3481 MetaScanner.metaScan(connection, visitor);
3482 }
3483
3484 errors.print("");
3485 return true;
3486 }
3487
3488
3489
3490
3491 static class MetaEntry extends HRegionInfo {
3492 ServerName regionServer;
3493 long modTime;
3494 HRegionInfo splitA, splitB;
3495
3496 public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3497 this(rinfo, regionServer, modTime, null, null);
3498 }
3499
3500 public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3501 HRegionInfo splitA, HRegionInfo splitB) {
3502 super(rinfo);
3503 this.regionServer = regionServer;
3504 this.modTime = modTime;
3505 this.splitA = splitA;
3506 this.splitB = splitB;
3507 }
3508
3509 @Override
3510 public boolean equals(Object o) {
3511 boolean superEq = super.equals(o);
3512 if (!superEq) {
3513 return superEq;
3514 }
3515
3516 MetaEntry me = (MetaEntry) o;
3517 if (!regionServer.equals(me.regionServer)) {
3518 return false;
3519 }
3520 return (modTime == me.modTime);
3521 }
3522
3523 @Override
3524 public int hashCode() {
3525 int hash = Arrays.hashCode(getRegionName());
3526 hash ^= getRegionId();
3527 hash ^= Arrays.hashCode(getStartKey());
3528 hash ^= Arrays.hashCode(getEndKey());
3529 hash ^= Boolean.valueOf(isOffline()).hashCode();
3530 hash ^= getTable().hashCode();
3531 if (regionServer != null) {
3532 hash ^= regionServer.hashCode();
3533 }
3534 hash ^= modTime;
3535 return hash;
3536 }
3537 }
3538
3539
3540
3541
3542 static class HdfsEntry {
3543 HRegionInfo hri;
3544 Path hdfsRegionDir = null;
3545 long hdfsRegionDirModTime = 0;
3546 boolean hdfsRegioninfoFilePresent = false;
3547 boolean hdfsOnlyEdits = false;
3548 }
3549
3550
3551
3552
3553 static class OnlineEntry {
3554 HRegionInfo hri;
3555 ServerName hsa;
3556
3557 @Override
3558 public String toString() {
3559 return hsa.toString() + ";" + hri.getRegionNameAsString();
3560 }
3561 }
3562
3563
3564
3565
3566
3567 public static class HbckInfo implements KeyRange {
3568 private MetaEntry metaEntry = null;
3569 private HdfsEntry hdfsEntry = null;
3570 private List<OnlineEntry> deployedEntries = Lists.newArrayList();
3571 private List<ServerName> deployedOn = Lists.newArrayList();
3572 private boolean skipChecks = false;
3573 private boolean isMerged = false;
3574 private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3575 private HRegionInfo primaryHRIForDeployedReplica = null;
3576
3577 HbckInfo(MetaEntry metaEntry) {
3578 this.metaEntry = metaEntry;
3579 }
3580
3581 public synchronized int getReplicaId() {
3582 return metaEntry != null? metaEntry.getReplicaId(): deployedReplicaId;
3583 }
3584
3585 public synchronized void addServer(HRegionInfo hri, ServerName server) {
3586 OnlineEntry rse = new OnlineEntry() ;
3587 rse.hri = hri;
3588 rse.hsa = server;
3589 this.deployedEntries.add(rse);
3590 this.deployedOn.add(server);
3591
3592 this.deployedReplicaId = hri.getReplicaId();
3593 this.primaryHRIForDeployedReplica =
3594 RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3595 }
3596
3597 @Override
3598 public synchronized String toString() {
3599 StringBuilder sb = new StringBuilder();
3600 sb.append("{ meta => ");
3601 sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3602 sb.append( ", hdfs => " + getHdfsRegionDir());
3603 sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3604 sb.append( ", replicaId => " + getReplicaId());
3605 sb.append(" }");
3606 return sb.toString();
3607 }
3608
3609 @Override
3610 public byte[] getStartKey() {
3611 if (this.metaEntry != null) {
3612 return this.metaEntry.getStartKey();
3613 } else if (this.hdfsEntry != null) {
3614 return this.hdfsEntry.hri.getStartKey();
3615 } else {
3616 LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3617 return null;
3618 }
3619 }
3620
3621 @Override
3622 public byte[] getEndKey() {
3623 if (this.metaEntry != null) {
3624 return this.metaEntry.getEndKey();
3625 } else if (this.hdfsEntry != null) {
3626 return this.hdfsEntry.hri.getEndKey();
3627 } else {
3628 LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3629 return null;
3630 }
3631 }
3632
3633 public TableName getTableName() {
3634 if (this.metaEntry != null) {
3635 return this.metaEntry.getTable();
3636 } else if (this.hdfsEntry != null) {
3637
3638
3639 Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3640 return FSUtils.getTableName(tableDir);
3641 } else {
3642
3643 for (OnlineEntry e : deployedEntries) {
3644 return e.hri.getTable();
3645 }
3646 return null;
3647 }
3648 }
3649
3650 public String getRegionNameAsString() {
3651 if (metaEntry != null) {
3652 return metaEntry.getRegionNameAsString();
3653 } else if (hdfsEntry != null) {
3654 if (hdfsEntry.hri != null) {
3655 return hdfsEntry.hri.getRegionNameAsString();
3656 }
3657 } else {
3658
3659 for (OnlineEntry e : deployedEntries) {
3660 return e.hri.getRegionNameAsString();
3661 }
3662 }
3663 return null;
3664 }
3665
3666 public byte[] getRegionName() {
3667 if (metaEntry != null) {
3668 return metaEntry.getRegionName();
3669 } else if (hdfsEntry != null) {
3670 return hdfsEntry.hri.getRegionName();
3671 } else {
3672
3673 for (OnlineEntry e : deployedEntries) {
3674 return e.hri.getRegionName();
3675 }
3676 return null;
3677 }
3678 }
3679
3680 public HRegionInfo getPrimaryHRIForDeployedReplica() {
3681 return primaryHRIForDeployedReplica;
3682 }
3683
3684 Path getHdfsRegionDir() {
3685 if (hdfsEntry == null) {
3686 return null;
3687 }
3688 return hdfsEntry.hdfsRegionDir;
3689 }
3690
3691 boolean containsOnlyHdfsEdits() {
3692 if (hdfsEntry == null) {
3693 return false;
3694 }
3695 return hdfsEntry.hdfsOnlyEdits;
3696 }
3697
3698 boolean isHdfsRegioninfoPresent() {
3699 if (hdfsEntry == null) {
3700 return false;
3701 }
3702 return hdfsEntry.hdfsRegioninfoFilePresent;
3703 }
3704
3705 long getModTime() {
3706 if (hdfsEntry == null) {
3707 return 0;
3708 }
3709 return hdfsEntry.hdfsRegionDirModTime;
3710 }
3711
3712 HRegionInfo getHdfsHRI() {
3713 if (hdfsEntry == null) {
3714 return null;
3715 }
3716 return hdfsEntry.hri;
3717 }
3718
3719 public void setSkipChecks(boolean skipChecks) {
3720 this.skipChecks = skipChecks;
3721 }
3722
3723 public boolean isSkipChecks() {
3724 return skipChecks;
3725 }
3726
3727 public void setMerged(boolean isMerged) {
3728 this.isMerged = isMerged;
3729 }
3730
3731 public boolean isMerged() {
3732 return this.isMerged;
3733 }
3734 }
3735
3736 final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3737 @Override
3738 public int compare(HbckInfo l, HbckInfo r) {
3739 if (l == r) {
3740
3741 return 0;
3742 }
3743
3744 int tableCompare = l.getTableName().compareTo(r.getTableName());
3745 if (tableCompare != 0) {
3746 return tableCompare;
3747 }
3748
3749 int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3750 l.getStartKey(), r.getStartKey());
3751 if (startComparison != 0) {
3752 return startComparison;
3753 }
3754
3755
3756 byte[] endKey = r.getEndKey();
3757 endKey = (endKey.length == 0) ? null : endKey;
3758 byte[] endKey2 = l.getEndKey();
3759 endKey2 = (endKey2.length == 0) ? null : endKey2;
3760 int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3761 endKey2, endKey);
3762
3763 if (endComparison != 0) {
3764 return endComparison;
3765 }
3766
3767
3768
3769 if (l.hdfsEntry == null && r.hdfsEntry == null) {
3770 return 0;
3771 }
3772 if (l.hdfsEntry == null && r.hdfsEntry != null) {
3773 return 1;
3774 }
3775
3776 if (r.hdfsEntry == null) {
3777 return -1;
3778 }
3779
3780 return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3781 }
3782 };
3783
3784
3785
3786
3787 private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3788 StringBuilder sb = new StringBuilder();
3789 int numOfSkippedRegions;
3790 errors.print("Summary:");
3791 for (TableInfo tInfo : tablesInfo.values()) {
3792 numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
3793 skippedRegions.get(tInfo.getName()).size() : 0;
3794
3795 if (errors.tableHasErrors(tInfo)) {
3796 errors.print("Table " + tInfo.getName() + " is inconsistent.");
3797 } else if (numOfSkippedRegions > 0){
3798 errors.print("Table " + tInfo.getName() + " is okay (with "
3799 + numOfSkippedRegions + " skipped regions).");
3800 }
3801 else {
3802 errors.print("Table " + tInfo.getName() + " is okay.");
3803 }
3804 errors.print(" Number of regions: " + tInfo.getNumRegions());
3805 if (numOfSkippedRegions > 0) {
3806 Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
3807 System.out.println(" Number of skipped regions: " + numOfSkippedRegions);
3808 System.out.println(" List of skipped regions:");
3809 for(String sr : skippedRegionStrings) {
3810 System.out.println(" " + sr);
3811 }
3812 }
3813 sb.setLength(0);
3814 sb.append(" Deployed on: ");
3815 for (ServerName server : tInfo.deployedOn) {
3816 sb.append(" " + server.toString());
3817 }
3818 errors.print(sb.toString());
3819 }
3820 }
3821
3822 static ErrorReporter getErrorReporter(
3823 final Configuration conf) throws ClassNotFoundException {
3824 Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3825 return ReflectionUtils.newInstance(reporter, conf);
3826 }
3827
3828 public interface ErrorReporter {
3829 enum ERROR_CODE {
3830 UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3831 NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3832 MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3833 FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3834 HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3835 ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3836 WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, ORPHANED_ZK_TABLE_ENTRY, BOUNDARIES_ERROR,
3837 UNDELETED_REPLICATION_QUEUE
3838 }
3839 void clear();
3840 void report(String message);
3841 void reportError(String message);
3842 void reportError(ERROR_CODE errorCode, String message);
3843 void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3844 void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3845 void reportError(
3846 ERROR_CODE errorCode,
3847 String message,
3848 TableInfo table,
3849 HbckInfo info1,
3850 HbckInfo info2
3851 );
3852 int summarize();
3853 void detail(String details);
3854 ArrayList<ERROR_CODE> getErrorList();
3855 void progress();
3856 void print(String message);
3857 void resetErrors();
3858 boolean tableHasErrors(TableInfo table);
3859 }
3860
3861 static class PrintingErrorReporter implements ErrorReporter {
3862 public int errorCount = 0;
3863 private int showProgress;
3864
3865 private static final int progressThreshold = 100;
3866
3867 Set<TableInfo> errorTables = new HashSet<TableInfo>();
3868
3869
3870 private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3871
3872 @Override
3873 public void clear() {
3874 errorTables.clear();
3875 errorList.clear();
3876 errorCount = 0;
3877 }
3878
3879 @Override
3880 public synchronized void reportError(ERROR_CODE errorCode, String message) {
3881 if (errorCode == ERROR_CODE.WRONG_USAGE) {
3882 System.err.println(message);
3883 return;
3884 }
3885
3886 errorList.add(errorCode);
3887 if (!getSUMMARY()) {
3888 System.out.println("ERROR: " + message);
3889 }
3890 errorCount++;
3891 showProgress = 0;
3892 }
3893
3894 @Override
3895 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3896 errorTables.add(table);
3897 reportError(errorCode, message);
3898 }
3899
3900 @Override
3901 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3902 HbckInfo info) {
3903 errorTables.add(table);
3904 String reference = "(region " + info.getRegionNameAsString() + ")";
3905 reportError(errorCode, reference + " " + message);
3906 }
3907
3908 @Override
3909 public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3910 HbckInfo info1, HbckInfo info2) {
3911 errorTables.add(table);
3912 String reference = "(regions " + info1.getRegionNameAsString()
3913 + " and " + info2.getRegionNameAsString() + ")";
3914 reportError(errorCode, reference + " " + message);
3915 }
3916
3917 @Override
3918 public synchronized void reportError(String message) {
3919 reportError(ERROR_CODE.UNKNOWN, message);
3920 }
3921
3922
3923
3924
3925
3926
3927 @Override
3928 public synchronized void report(String message) {
3929 if (!getSUMMARY()) {
3930 System.out.println("ERROR: " + message);
3931 }
3932 showProgress = 0;
3933 }
3934
3935 @Override
3936 public synchronized int summarize() {
3937 System.out.println(Integer.toString(errorCount) +
3938 " inconsistencies detected.");
3939 if (errorCount == 0) {
3940 System.out.println("Status: OK");
3941 return 0;
3942 } else {
3943 System.out.println("Status: INCONSISTENT");
3944 return -1;
3945 }
3946 }
3947
3948 @Override
3949 public ArrayList<ERROR_CODE> getErrorList() {
3950 return errorList;
3951 }
3952
3953 @Override
3954 public synchronized void print(String message) {
3955 if (!getSUMMARY()) {
3956 System.out.println(message);
3957 }
3958 }
3959
3960 private synchronized static boolean getSUMMARY() {
3961 return SUMMARY;
3962 }
3963
3964 @Override
3965 public boolean tableHasErrors(TableInfo table) {
3966 return errorTables.contains(table);
3967 }
3968
3969 @Override
3970 public void resetErrors() {
3971 errorCount = 0;
3972 }
3973
3974 @Override
3975 public synchronized void detail(String message) {
3976 if (details) {
3977 System.out.println(message);
3978 }
3979 showProgress = 0;
3980 }
3981
3982 @Override
3983 public synchronized void progress() {
3984 if (showProgress++ == progressThreshold) {
3985 if (!getSUMMARY()) {
3986 System.out.print(".");
3987 }
3988 showProgress = 0;
3989 }
3990 }
3991 }
3992
3993
3994
3995
3996 static class WorkItemRegion implements Callable<Void> {
3997 private HBaseFsck hbck;
3998 private ServerName rsinfo;
3999 private ErrorReporter errors;
4000 private HConnection connection;
4001
4002 WorkItemRegion(HBaseFsck hbck, ServerName info,
4003 ErrorReporter errors, HConnection connection) {
4004 this.hbck = hbck;
4005 this.rsinfo = info;
4006 this.errors = errors;
4007 this.connection = connection;
4008 }
4009
4010 @Override
4011 public synchronized Void call() throws IOException {
4012 errors.progress();
4013 try {
4014 BlockingInterface server = connection.getAdmin(rsinfo);
4015
4016
4017 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
4018 regions = filterRegions(regions);
4019
4020 if (details) {
4021 errors.detail("RegionServer: " + rsinfo.getServerName() +
4022 " number of regions: " + regions.size());
4023 for (HRegionInfo rinfo: regions) {
4024 errors.detail(" " + rinfo.getRegionNameAsString() +
4025 " id: " + rinfo.getRegionId() +
4026 " encoded_name: " + rinfo.getEncodedName() +
4027 " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
4028 " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
4029 }
4030 }
4031
4032
4033 for (HRegionInfo r:regions) {
4034 HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
4035 hbi.addServer(r, rsinfo);
4036 }
4037 } catch (IOException e) {
4038 errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
4039 " Unable to fetch region information. " + e);
4040 throw e;
4041 }
4042 return null;
4043 }
4044
4045 private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
4046 List<HRegionInfo> ret = Lists.newArrayList();
4047 for (HRegionInfo hri : regions) {
4048 if (hri.isMetaTable() || (!hbck.checkMetaOnly
4049 && hbck.isTableIncluded(hri.getTable()))) {
4050 ret.add(hri);
4051 }
4052 }
4053 return ret;
4054 }
4055 }
4056
4057
4058
4059
4060
4061 static class WorkItemHdfsDir implements Callable<Void> {
4062 private HBaseFsck hbck;
4063 private FileStatus tableDir;
4064 private ErrorReporter errors;
4065 private FileSystem fs;
4066
4067 WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
4068 FileStatus status) {
4069 this.hbck = hbck;
4070 this.fs = fs;
4071 this.tableDir = status;
4072 this.errors = errors;
4073 }
4074
4075 @Override
4076 public synchronized Void call() throws IOException {
4077 try {
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134 static class WorkItemHdfsRegionInfo implements Callable<Void> {
4135 private HbckInfo hbi;
4136 private HBaseFsck hbck;
4137 private ErrorReporter errors;
4138
4139 WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
4140 this.hbi = hbi;
4141 this.hbck = hbck;
4142 this.errors = errors;
4143 }
4144
4145 @Override
4146 public synchronized Void call() throws IOException {
4147
4148 if (hbi.getHdfsHRI() == null) {
4149 try {
4150 errors.progress();
4151 hbck.loadHdfsRegioninfo(hbi);
4152 } catch (IOException ioe) {
4153 String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
4154 + hbi.getTableName() + " in hdfs dir "
4155 + hbi.getHdfsRegionDir()
4156 + "! It may be an invalid format or version file. Treating as "
4157 + "an orphaned regiondir.";
4158 errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
4159 try {
4160 hbck.debugLsr(hbi.getHdfsRegionDir());
4161 } catch (IOException ioe2) {
4162 LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
4163 throw ioe2;
4164 }
4165 hbck.orphanHdfsDirs.add(hbi);
4166 throw ioe;
4167 }
4168 }
4169 return null;
4170 }
4171 };
4172
4173
4174
4175
4176
4177 public static void setDisplayFullReport() {
4178 details = true;
4179 }
4180
4181
4182
4183
4184 public static void setForceExclusive() {
4185 forceExclusive = true;
4186 }
4187
4188
4189
4190
4191 public boolean isExclusive() {
4192 return fixAny || forceExclusive;
4193 }
4194
4195
4196
4197
4198 public static void setDisableBalancer() {
4199 disableBalancer = true;
4200 }
4201
4202
4203
4204
4205
4206
4207 public boolean shouldDisableBalancer() {
4208 return fixAny || disableBalancer;
4209 }
4210
4211
4212
4213
4214
4215 synchronized static void setSummary() {
4216 SUMMARY = true;
4217 }
4218
4219
4220
4221
4222
4223 void setCheckMetaOnly() {
4224 checkMetaOnly = true;
4225 }
4226
4227
4228
4229
4230 void setRegionBoundariesCheck() {
4231 checkRegionBoundaries = true;
4232 }
4233
4234
4235
4236
4237
4238 public void setFixTableLocks(boolean shouldFix) {
4239 fixTableLocks = shouldFix;
4240 fixAny |= shouldFix;
4241 }
4242
4243
4244
4245
4246 public void setFixReplication(boolean shouldFix) {
4247 fixReplication = shouldFix;
4248 fixAny |= shouldFix;
4249 }
4250
4251
4252
4253
4254
4255 public void setFixTableZNodes(boolean shouldFix) {
4256 fixTableZNodes = shouldFix;
4257 fixAny |= shouldFix;
4258 }
4259
4260
4261
4262
4263
4264
4265
4266 void setShouldRerun() {
4267 rerun = true;
4268 }
4269
4270 boolean shouldRerun() {
4271 return rerun;
4272 }
4273
4274
4275
4276
4277
4278 public void setFixAssignments(boolean shouldFix) {
4279 fixAssignments = shouldFix;
4280 fixAny |= shouldFix;
4281 }
4282
4283 boolean shouldFixAssignments() {
4284 return fixAssignments;
4285 }
4286
4287 public void setFixMeta(boolean shouldFix) {
4288 fixMeta = shouldFix;
4289 fixAny |= shouldFix;
4290 }
4291
4292 boolean shouldFixMeta() {
4293 return fixMeta;
4294 }
4295
4296 public void setFixEmptyMetaCells(boolean shouldFix) {
4297 fixEmptyMetaCells = shouldFix;
4298 fixAny |= shouldFix;
4299 }
4300
4301 boolean shouldFixEmptyMetaCells() {
4302 return fixEmptyMetaCells;
4303 }
4304
4305 public void setCheckHdfs(boolean checking) {
4306 checkHdfs = checking;
4307 }
4308
4309 boolean shouldCheckHdfs() {
4310 return checkHdfs;
4311 }
4312
4313 public void setFixHdfsHoles(boolean shouldFix) {
4314 fixHdfsHoles = shouldFix;
4315 fixAny |= shouldFix;
4316 }
4317
4318 boolean shouldFixHdfsHoles() {
4319 return fixHdfsHoles;
4320 }
4321
4322 public void setFixTableOrphans(boolean shouldFix) {
4323 fixTableOrphans = shouldFix;
4324 fixAny |= shouldFix;
4325 }
4326
4327 boolean shouldFixTableOrphans() {
4328 return fixTableOrphans;
4329 }
4330
4331 public void setFixHdfsOverlaps(boolean shouldFix) {
4332 fixHdfsOverlaps = shouldFix;
4333 fixAny |= shouldFix;
4334 }
4335
4336 boolean shouldFixHdfsOverlaps() {
4337 return fixHdfsOverlaps;
4338 }
4339
4340 public void setFixHdfsOrphans(boolean shouldFix) {
4341 fixHdfsOrphans = shouldFix;
4342 fixAny |= shouldFix;
4343 }
4344
4345 boolean shouldFixHdfsOrphans() {
4346 return fixHdfsOrphans;
4347 }
4348
4349 public void setFixVersionFile(boolean shouldFix) {
4350 fixVersionFile = shouldFix;
4351 fixAny |= shouldFix;
4352 }
4353
4354 public boolean shouldFixVersionFile() {
4355 return fixVersionFile;
4356 }
4357
4358 public void setSidelineBigOverlaps(boolean sbo) {
4359 this.sidelineBigOverlaps = sbo;
4360 }
4361
4362 public boolean shouldSidelineBigOverlaps() {
4363 return sidelineBigOverlaps;
4364 }
4365
4366 public void setFixSplitParents(boolean shouldFix) {
4367 fixSplitParents = shouldFix;
4368 fixAny |= shouldFix;
4369 }
4370
4371 boolean shouldFixSplitParents() {
4372 return fixSplitParents;
4373 }
4374
4375 public void setFixReferenceFiles(boolean shouldFix) {
4376 fixReferenceFiles = shouldFix;
4377 fixAny |= shouldFix;
4378 }
4379
4380 boolean shouldFixReferenceFiles() {
4381 return fixReferenceFiles;
4382 }
4383
4384 public boolean shouldIgnorePreCheckPermission() {
4385 return !fixAny || ignorePreCheckPermission;
4386 }
4387
4388 public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4389 this.ignorePreCheckPermission = ignorePreCheckPermission;
4390 }
4391
4392
4393
4394
4395 public void setMaxMerge(int mm) {
4396 this.maxMerge = mm;
4397 }
4398
4399 public int getMaxMerge() {
4400 return maxMerge;
4401 }
4402
4403 public void setMaxOverlapsToSideline(int mo) {
4404 this.maxOverlapsToSideline = mo;
4405 }
4406
4407 public int getMaxOverlapsToSideline() {
4408 return maxOverlapsToSideline;
4409 }
4410
4411
4412
4413
4414
4415 boolean isTableIncluded(TableName table) {
4416 return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4417 }
4418
4419 public void includeTable(TableName table) {
4420 tablesIncluded.add(table);
4421 }
4422
4423 Set<TableName> getIncludedTables() {
4424 return new HashSet<TableName>(tablesIncluded);
4425 }
4426
4427
4428
4429
4430
4431
4432 public void setTimeLag(long seconds) {
4433 timelag = seconds * 1000;
4434 }
4435
4436
4437
4438
4439
4440 public void setSidelineDir(String sidelineDir) {
4441 this.sidelineDir = new Path(sidelineDir);
4442 }
4443
4444 protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4445 return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4446 }
4447
4448 public HFileCorruptionChecker getHFilecorruptionChecker() {
4449 return hfcc;
4450 }
4451
4452 public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4453 this.hfcc = hfcc;
4454 }
4455
4456 public void setRetCode(int code) {
4457 this.retcode = code;
4458 }
4459
4460 public int getRetCode() {
4461 return retcode;
4462 }
4463
4464 protected HBaseFsck printUsageAndExit() {
4465 StringWriter sw = new StringWriter(2048);
4466 PrintWriter out = new PrintWriter(sw);
4467 out.println("Usage: fsck [opts] {only tables}");
4468 out.println(" where [opts] are:");
4469 out.println(" -help Display help options (this)");
4470 out.println(" -details Display full report of all regions.");
4471 out.println(" -timelag <timeInSeconds> Process only regions that " +
4472 " have not experienced any metadata updates in the last " +
4473 " <timeInSeconds> seconds.");
4474 out.println(" -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4475 " before checking if the fix worked if run with -fix");
4476 out.println(" -summary Print only summary of the tables and status.");
4477 out.println(" -metaonly Only check the state of the hbase:meta table.");
4478 out.println(" -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4479 out.println(" -boundaries Verify that regions boundaries are the same between META and store files.");
4480 out.println(" -exclusive Abort if another hbck is exclusive or fixing.");
4481 out.println(" -disableBalancer Disable the load balancer.");
4482
4483 out.println("");
4484 out.println(" Metadata Repair options: (expert features, use with caution!)");
4485 out.println(" -fix Try to fix region assignments. This is for backwards compatiblity");
4486 out.println(" -fixAssignments Try to fix region assignments. Replaces the old -fix");
4487 out.println(" -fixMeta Try to fix meta problems. This assumes HDFS region info is good.");
4488 out.println(" -noHdfsChecking Don't load/check region info from HDFS."
4489 + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4490 out.println(" -fixHdfsHoles Try to fix region holes in hdfs.");
4491 out.println(" -fixHdfsOrphans Try to fix region dirs with no .regioninfo file in hdfs");
4492 out.println(" -fixTableOrphans Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4493 out.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs.");
4494 out.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs.");
4495 out.println(" -maxMerge <n> When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4496 out.println(" -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps");
4497 out.println(" -maxOverlapsToSideline <n> When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4498 out.println(" -fixSplitParents Try to force offline split parents to be online.");
4499 out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check");
4500 out.println(" -fixReferenceFiles Try to offline lingering reference store files");
4501 out.println(" -fixEmptyMetaCells Try to fix hbase:meta entries not referencing any region"
4502 + " (empty REGIONINFO_QUALIFIER rows)");
4503
4504 out.println("");
4505 out.println(" Datafile Repair options: (expert features, use with caution!)");
4506 out.println(" -checkCorruptHFiles Check all Hfiles by opening them to make sure they are valid");
4507 out.println(" -sidelineCorruptHFiles Quarantine corrupted HFiles. implies -checkCorruptHFiles");
4508
4509 out.println("");
4510 out.println(" Metadata Repair shortcuts");
4511 out.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4512 "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps " +
4513 "-fixReferenceFiles -fixTableLocks -fixOrphanedTableZnodes");
4514 out.println(" -repairHoles Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4515
4516 out.println("");
4517 out.println(" Table lock options");
4518 out.println(" -fixTableLocks Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4519
4520 out.println("");
4521 out.println(" Table Znode options");
4522 out.println(" -fixOrphanedTableZnodes Set table state in ZNode to disabled if table does not exists");
4523
4524 out.println("");
4525 out.println(" Replication options");
4526 out.println(" -fixReplication Deletes replication queues for removed peers");
4527
4528 out.flush();
4529 errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4530
4531 setRetCode(-2);
4532 return this;
4533 }
4534
4535
4536
4537
4538
4539
4540
4541 public static void main(String[] args) throws Exception {
4542
4543 Configuration conf = HBaseConfiguration.create();
4544 Path hbasedir = FSUtils.getRootDir(conf);
4545 URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4546 FSUtils.setFsDefault(conf, new Path(defaultFs));
4547 int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4548 System.exit(ret);
4549 }
4550
4551
4552
4553
4554 static class HBaseFsckTool extends Configured implements Tool {
4555 HBaseFsckTool(Configuration conf) { super(conf); }
4556 @Override
4557 public int run(String[] args) throws Exception {
4558 HBaseFsck hbck = new HBaseFsck(getConf());
4559 hbck.exec(hbck.executor, args);
4560 hbck.close();
4561 return hbck.getRetCode();
4562 }
4563 };
4564
4565
4566 public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4567 ServiceException, InterruptedException {
4568 long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4569
4570 boolean checkCorruptHFiles = false;
4571 boolean sidelineCorruptHFiles = false;
4572
4573
4574 for (int i = 0; i < args.length; i++) {
4575 String cmd = args[i];
4576 if (cmd.equals("-help") || cmd.equals("-h")) {
4577 return printUsageAndExit();
4578 } else if (cmd.equals("-details")) {
4579 setDisplayFullReport();
4580 } else if (cmd.equals("-exclusive")) {
4581 setForceExclusive();
4582 } else if (cmd.equals("-disableBalancer")) {
4583 setDisableBalancer();
4584 } else if (cmd.equals("-timelag")) {
4585 if (i == args.length - 1) {
4586 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4587 return printUsageAndExit();
4588 }
4589 try {
4590 long timelag = Long.parseLong(args[i+1]);
4591 setTimeLag(timelag);
4592 } catch (NumberFormatException e) {
4593 errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4594 return printUsageAndExit();
4595 }
4596 i++;
4597 } else if (cmd.equals("-sleepBeforeRerun")) {
4598 if (i == args.length - 1) {
4599 errors.reportError(ERROR_CODE.WRONG_USAGE,
4600 "HBaseFsck: -sleepBeforeRerun needs a value.");
4601 return printUsageAndExit();
4602 }
4603 try {
4604 sleepBeforeRerun = Long.parseLong(args[i+1]);
4605 } catch (NumberFormatException e) {
4606 errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4607 return printUsageAndExit();
4608 }
4609 i++;
4610 } else if (cmd.equals("-sidelineDir")) {
4611 if (i == args.length - 1) {
4612 errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4613 return printUsageAndExit();
4614 }
4615 i++;
4616 setSidelineDir(args[i]);
4617 } else if (cmd.equals("-fix")) {
4618 errors.reportError(ERROR_CODE.WRONG_USAGE,
4619 "This option is deprecated, please use -fixAssignments instead.");
4620 setFixAssignments(true);
4621 } else if (cmd.equals("-fixAssignments")) {
4622 setFixAssignments(true);
4623 } else if (cmd.equals("-fixMeta")) {
4624 setFixMeta(true);
4625 } else if (cmd.equals("-noHdfsChecking")) {
4626 setCheckHdfs(false);
4627 } else if (cmd.equals("-fixHdfsHoles")) {
4628 setFixHdfsHoles(true);
4629 } else if (cmd.equals("-fixHdfsOrphans")) {
4630 setFixHdfsOrphans(true);
4631 } else if (cmd.equals("-fixTableOrphans")) {
4632 setFixTableOrphans(true);
4633 } else if (cmd.equals("-fixHdfsOverlaps")) {
4634 setFixHdfsOverlaps(true);
4635 } else if (cmd.equals("-fixVersionFile")) {
4636 setFixVersionFile(true);
4637 } else if (cmd.equals("-sidelineBigOverlaps")) {
4638 setSidelineBigOverlaps(true);
4639 } else if (cmd.equals("-fixSplitParents")) {
4640 setFixSplitParents(true);
4641 } else if (cmd.equals("-ignorePreCheckPermission")) {
4642 setIgnorePreCheckPermission(true);
4643 } else if (cmd.equals("-checkCorruptHFiles")) {
4644 checkCorruptHFiles = true;
4645 } else if (cmd.equals("-sidelineCorruptHFiles")) {
4646 sidelineCorruptHFiles = true;
4647 } else if (cmd.equals("-fixReferenceFiles")) {
4648 setFixReferenceFiles(true);
4649 } else if (cmd.equals("-fixEmptyMetaCells")) {
4650 setFixEmptyMetaCells(true);
4651 } else if (cmd.equals("-repair")) {
4652
4653
4654 setFixHdfsHoles(true);
4655 setFixHdfsOrphans(true);
4656 setFixMeta(true);
4657 setFixAssignments(true);
4658 setFixHdfsOverlaps(true);
4659 setFixVersionFile(true);
4660 setSidelineBigOverlaps(true);
4661 setFixSplitParents(false);
4662 setCheckHdfs(true);
4663 setFixReferenceFiles(true);
4664 setFixTableLocks(true);
4665 setFixTableZNodes(true);
4666 } else if (cmd.equals("-repairHoles")) {
4667
4668 setFixHdfsHoles(true);
4669 setFixHdfsOrphans(false);
4670 setFixMeta(true);
4671 setFixAssignments(true);
4672 setFixHdfsOverlaps(false);
4673 setSidelineBigOverlaps(false);
4674 setFixSplitParents(false);
4675 setCheckHdfs(true);
4676 } else if (cmd.equals("-maxOverlapsToSideline")) {
4677 if (i == args.length - 1) {
4678 errors.reportError(ERROR_CODE.WRONG_USAGE,
4679 "-maxOverlapsToSideline needs a numeric value argument.");
4680 return printUsageAndExit();
4681 }
4682 try {
4683 int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4684 setMaxOverlapsToSideline(maxOverlapsToSideline);
4685 } catch (NumberFormatException e) {
4686 errors.reportError(ERROR_CODE.WRONG_USAGE,
4687 "-maxOverlapsToSideline needs a numeric value argument.");
4688 return printUsageAndExit();
4689 }
4690 i++;
4691 } else if (cmd.equals("-maxMerge")) {
4692 if (i == args.length - 1) {
4693 errors.reportError(ERROR_CODE.WRONG_USAGE,
4694 "-maxMerge needs a numeric value argument.");
4695 return printUsageAndExit();
4696 }
4697 try {
4698 int maxMerge = Integer.parseInt(args[i+1]);
4699 setMaxMerge(maxMerge);
4700 } catch (NumberFormatException e) {
4701 errors.reportError(ERROR_CODE.WRONG_USAGE,
4702 "-maxMerge needs a numeric value argument.");
4703 return printUsageAndExit();
4704 }
4705 i++;
4706 } else if (cmd.equals("-summary")) {
4707 setSummary();
4708 } else if (cmd.equals("-metaonly")) {
4709 setCheckMetaOnly();
4710 } else if (cmd.equals("-boundaries")) {
4711 setRegionBoundariesCheck();
4712 } else if (cmd.equals("-fixTableLocks")) {
4713 setFixTableLocks(true);
4714 } else if (cmd.equals("-fixReplication")) {
4715 setFixReplication(true);
4716 } else if (cmd.equals("-fixOrphanedTableZnodes")) {
4717 setFixTableZNodes(true);
4718 } else if (cmd.startsWith("-")) {
4719 errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4720 return printUsageAndExit();
4721 } else {
4722 includeTable(TableName.valueOf(cmd));
4723 errors.print("Allow checking/fixes for table: " + cmd);
4724 }
4725 }
4726
4727 errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4728
4729
4730 try {
4731 preCheckPermission();
4732 } catch (AccessDeniedException ace) {
4733 Runtime.getRuntime().exit(-1);
4734 } catch (IOException ioe) {
4735 Runtime.getRuntime().exit(-1);
4736 }
4737
4738
4739 connect();
4740
4741 try {
4742
4743 if (checkCorruptHFiles || sidelineCorruptHFiles) {
4744 LOG.info("Checking all hfiles for corruption");
4745 HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4746 setHFileCorruptionChecker(hfcc);
4747 Collection<TableName> tables = getIncludedTables();
4748 Collection<Path> tableDirs = new ArrayList<Path>();
4749 Path rootdir = FSUtils.getRootDir(getConf());
4750 if (tables.size() > 0) {
4751 for (TableName t : tables) {
4752 tableDirs.add(FSUtils.getTableDir(rootdir, t));
4753 }
4754 } else {
4755 tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4756 }
4757 hfcc.checkTables(tableDirs);
4758 hfcc.report(errors);
4759 }
4760
4761
4762 int code = onlineHbck();
4763 setRetCode(code);
4764
4765
4766
4767
4768 if (shouldRerun()) {
4769 try {
4770 LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4771 Thread.sleep(sleepBeforeRerun);
4772 } catch (InterruptedException ie) {
4773 LOG.warn("Interrupted while sleeping");
4774 return this;
4775 }
4776
4777 setFixAssignments(false);
4778 setFixMeta(false);
4779 setFixHdfsHoles(false);
4780 setFixHdfsOverlaps(false);
4781 setFixVersionFile(false);
4782 setFixTableOrphans(false);
4783 errors.resetErrors();
4784 code = onlineHbck();
4785 setRetCode(code);
4786 }
4787 } finally {
4788 IOUtils.closeQuietly(this);
4789 }
4790 return this;
4791 }
4792
4793
4794
4795
4796 void debugLsr(Path p) throws IOException {
4797 debugLsr(getConf(), p, errors);
4798 }
4799
4800
4801
4802
4803 public static void debugLsr(Configuration conf,
4804 Path p) throws IOException {
4805 debugLsr(conf, p, new PrintingErrorReporter());
4806 }
4807
4808
4809
4810
4811 public static void debugLsr(Configuration conf,
4812 Path p, ErrorReporter errors) throws IOException {
4813 if (!LOG.isDebugEnabled() || p == null) {
4814 return;
4815 }
4816 FileSystem fs = p.getFileSystem(conf);
4817
4818 if (!fs.exists(p)) {
4819
4820 return;
4821 }
4822 errors.print(p.toString());
4823
4824 if (fs.isFile(p)) {
4825 return;
4826 }
4827
4828 if (fs.getFileStatus(p).isDirectory()) {
4829 FileStatus[] fss= fs.listStatus(p);
4830 for (FileStatus status : fss) {
4831 debugLsr(conf, status.getPath(), errors);
4832 }
4833 }
4834 }
4835 }