View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.EOFException;
22  import java.io.FileNotFoundException;
23  import java.io.IOException;
24  import java.io.InterruptedIOException;
25  import java.lang.reflect.Constructor;
26  import java.text.ParseException;
27  import java.util.AbstractList;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.Comparator;
33  import java.util.HashMap;
34  import java.util.HashSet;
35  import java.util.Iterator;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Map.Entry;
39  import java.util.NavigableMap;
40  import java.util.NavigableSet;
41  import java.util.RandomAccess;
42  import java.util.Set;
43  import java.util.TreeMap;
44  import java.util.concurrent.Callable;
45  import java.util.concurrent.CompletionService;
46  import java.util.concurrent.ConcurrentHashMap;
47  import java.util.concurrent.ConcurrentMap;
48  import java.util.concurrent.ConcurrentSkipListMap;
49  import java.util.concurrent.ExecutionException;
50  import java.util.concurrent.ExecutorCompletionService;
51  import java.util.concurrent.ExecutorService;
52  import java.util.concurrent.Executors;
53  import java.util.concurrent.Future;
54  import java.util.concurrent.FutureTask;
55  import java.util.concurrent.ThreadFactory;
56  import java.util.concurrent.ThreadPoolExecutor;
57  import java.util.concurrent.TimeUnit;
58  import java.util.concurrent.TimeoutException;
59  import java.util.concurrent.atomic.AtomicBoolean;
60  import java.util.concurrent.atomic.AtomicInteger;
61  import java.util.concurrent.atomic.AtomicLong;
62  import java.util.concurrent.locks.Lock;
63  import java.util.concurrent.locks.ReadWriteLock;
64  import java.util.concurrent.locks.ReentrantReadWriteLock;
65  
66  import org.apache.commons.lang.RandomStringUtils;
67  import org.apache.commons.logging.Log;
68  import org.apache.commons.logging.LogFactory;
69  import org.apache.hadoop.conf.Configuration;
70  import org.apache.hadoop.fs.FileStatus;
71  import org.apache.hadoop.fs.FileSystem;
72  import org.apache.hadoop.fs.Path;
73  import org.apache.hadoop.hbase.Cell;
74  import org.apache.hadoop.hbase.CellScanner;
75  import org.apache.hadoop.hbase.CellUtil;
76  import org.apache.hadoop.hbase.CompoundConfiguration;
77  import org.apache.hadoop.hbase.DoNotRetryIOException;
78  import org.apache.hadoop.hbase.DroppedSnapshotException;
79  import org.apache.hadoop.hbase.HBaseConfiguration;
80  import org.apache.hadoop.hbase.HColumnDescriptor;
81  import org.apache.hadoop.hbase.HConstants;
82  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
83  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
84  import org.apache.hadoop.hbase.HRegionInfo;
85  import org.apache.hadoop.hbase.HTableDescriptor;
86  import org.apache.hadoop.hbase.KeyValue;
87  import org.apache.hadoop.hbase.KeyValueUtil;
88  import org.apache.hadoop.hbase.NamespaceDescriptor;
89  import org.apache.hadoop.hbase.NotServingRegionException;
90  import org.apache.hadoop.hbase.RegionTooBusyException;
91  import org.apache.hadoop.hbase.TableName;
92  import org.apache.hadoop.hbase.Tag;
93  import org.apache.hadoop.hbase.TagType;
94  import org.apache.hadoop.hbase.UnknownScannerException;
95  import org.apache.hadoop.hbase.backup.HFileArchiver;
96  import org.apache.hadoop.hbase.classification.InterfaceAudience;
97  import org.apache.hadoop.hbase.client.Append;
98  import org.apache.hadoop.hbase.client.Delete;
99  import org.apache.hadoop.hbase.client.Durability;
100 import org.apache.hadoop.hbase.client.Get;
101 import org.apache.hadoop.hbase.client.Increment;
102 import org.apache.hadoop.hbase.client.IsolationLevel;
103 import org.apache.hadoop.hbase.client.Mutation;
104 import org.apache.hadoop.hbase.client.Put;
105 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
106 import org.apache.hadoop.hbase.client.Result;
107 import org.apache.hadoop.hbase.client.RowMutations;
108 import org.apache.hadoop.hbase.client.Scan;
109 import org.apache.hadoop.hbase.conf.ConfigurationManager;
110 import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver;
111 import org.apache.hadoop.hbase.coprocessor.RegionObserver;
112 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
113 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
114 import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException;
115 import org.apache.hadoop.hbase.exceptions.UnknownProtocolException;
116 import org.apache.hadoop.hbase.filter.ByteArrayComparable;
117 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
118 import org.apache.hadoop.hbase.filter.FilterWrapper;
119 import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
120 import org.apache.hadoop.hbase.io.HeapSize;
121 import org.apache.hadoop.hbase.io.TimeRange;
122 import org.apache.hadoop.hbase.io.hfile.BlockCache;
123 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
124 import org.apache.hadoop.hbase.io.hfile.HFile;
125 import org.apache.hadoop.hbase.ipc.CallerDisconnectedException;
126 import org.apache.hadoop.hbase.ipc.RpcCallContext;
127 import org.apache.hadoop.hbase.ipc.RpcServer;
128 import org.apache.hadoop.hbase.mob.MobUtils;
129 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
130 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
131 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
132 import org.apache.hadoop.hbase.protobuf.ResponseConverter;
133 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
134 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
135 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
136 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
137 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.StoreSequenceId;
138 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
139 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
140 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
141 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
142 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
143 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
144 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
145 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
146 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
147 import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.WriteEntry;
148 import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
149 import org.apache.hadoop.hbase.regionserver.ScannerContext.NextState;
150 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
151 import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
152 import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputControllerFactory;
153 import org.apache.hadoop.hbase.regionserver.compactions.NoLimitCompactionThroughputController;
154 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
155 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
156 import org.apache.hadoop.hbase.regionserver.wal.ReplayHLogKey;
157 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
158 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
159 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
160 import org.apache.hadoop.hbase.security.User;
161 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
162 import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
163 import org.apache.hadoop.hbase.util.ByteStringer;
164 import org.apache.hadoop.hbase.util.Bytes;
165 import org.apache.hadoop.hbase.util.CancelableProgressable;
166 import org.apache.hadoop.hbase.util.ClassSize;
167 import org.apache.hadoop.hbase.util.CompressionTest;
168 import org.apache.hadoop.hbase.util.Counter;
169 import org.apache.hadoop.hbase.util.EncryptionTest;
170 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
171 import org.apache.hadoop.hbase.util.FSTableDescriptors;
172 import org.apache.hadoop.hbase.util.FSUtils;
173 import org.apache.hadoop.hbase.util.HashedBytes;
174 import org.apache.hadoop.hbase.util.Pair;
175 import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
176 import org.apache.hadoop.hbase.util.Threads;
177 import org.apache.hadoop.hbase.wal.WAL;
178 import org.apache.hadoop.hbase.wal.WALFactory;
179 import org.apache.hadoop.hbase.wal.WALKey;
180 import org.apache.hadoop.hbase.wal.WALSplitter;
181 import org.apache.hadoop.hbase.wal.WALSplitter.MutationReplay;
182 import org.apache.hadoop.io.MultipleIOException;
183 import org.apache.hadoop.util.StringUtils;
184 import org.apache.htrace.Trace;
185 import org.apache.htrace.TraceScope;
186 
187 import com.google.common.annotations.VisibleForTesting;
188 import com.google.common.base.Optional;
189 import com.google.common.base.Preconditions;
190 import com.google.common.collect.Lists;
191 import com.google.common.collect.Maps;
192 import com.google.common.io.Closeables;
193 import com.google.protobuf.ByteString;
194 import com.google.protobuf.Descriptors;
195 import com.google.protobuf.Message;
196 import com.google.protobuf.RpcCallback;
197 import com.google.protobuf.RpcController;
198 import com.google.protobuf.Service;
199 import com.google.protobuf.TextFormat;
200 
201 @InterfaceAudience.Private
202 public class HRegion implements HeapSize, PropagatingConfigurationObserver, Region {
203   private static final Log LOG = LogFactory.getLog(HRegion.class);
204 
205   public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY =
206     "hbase.hregion.scan.loadColumnFamiliesOnDemand";
207 
208   /**
209    * Longest time we'll wait on a sequenceid.
210    * Sequenceid comes up out of the WAL subsystem. WAL subsystem can go bad or a test might use
211    * it without cleaning up previous usage properly; generally, a WAL roll is needed. The timeout
212    * is for a latch in WALKey. There is no global accounting of outstanding WALKeys; intentionally
213    * to avoid contention, but it makes it so if an abort or problem, we could be stuck waiting
214    * on the WALKey latch. Revisit.
215    */
216   private final int maxWaitForSeqId;
217   private static final String MAX_WAIT_FOR_SEQ_ID_KEY = "hbase.hregion.max.wait.for.sequenceid.ms";
218   private static final int DEFAULT_MAX_WAIT_FOR_SEQ_ID = 30000;
219 
220   /**
221    * This is the global default value for durability. All tables/mutations not
222    * defining a durability or using USE_DEFAULT will default to this value.
223    */
224   private static final Durability DEFAULT_DURABILITY = Durability.SYNC_WAL;
225 
226   final AtomicBoolean closed = new AtomicBoolean(false);
227 
228   /* Closing can take some time; use the closing flag if there is stuff we don't
229    * want to do while in closing state; e.g. like offer this region up to the
230    * master as a region to close if the carrying regionserver is overloaded.
231    * Once set, it is never cleared.
232    */
233   final AtomicBoolean closing = new AtomicBoolean(false);
234 
235   /**
236    * The max sequence id of flushed data on this region. There is no edit in memory that is
237    * less that this sequence id.
238    */
239   private volatile long maxFlushedSeqId = HConstants.NO_SEQNUM;
240 
241   /**
242    * Record the sequence id of last flush operation. Can be in advance of
243    * {@link #maxFlushedSeqId} when flushing a single column family. In this case,
244    * {@link #maxFlushedSeqId} will be older than the oldest edit in memory.
245    */
246   private volatile long lastFlushOpSeqId = HConstants.NO_SEQNUM;
247 
248   /**
249    * The sequence id of the last replayed open region event from the primary region. This is used
250    * to skip entries before this due to the possibility of replay edits coming out of order from
251    * replication.
252    */
253   protected volatile long lastReplayedOpenRegionSeqId = -1L;
254   protected volatile long lastReplayedCompactionSeqId = -1L;
255 
256   //////////////////////////////////////////////////////////////////////////////
257   // Members
258   //////////////////////////////////////////////////////////////////////////////
259 
260   // map from a locked row to the context for that lock including:
261   // - CountDownLatch for threads waiting on that row
262   // - the thread that owns the lock (allow reentrancy)
263   // - reference count of (reentrant) locks held by the thread
264   // - the row itself
265   private final ConcurrentHashMap<HashedBytes, RowLockContext> lockedRows =
266       new ConcurrentHashMap<HashedBytes, RowLockContext>();
267 
268   protected final Map<byte[], Store> stores = new ConcurrentSkipListMap<byte[], Store>(
269       Bytes.BYTES_RAWCOMPARATOR);
270 
271   // TODO: account for each registered handler in HeapSize computation
272   private Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
273 
274   private final AtomicLong memstoreSize = new AtomicLong(0);
275 
276   // Debug possible data loss due to WAL off
277   final Counter numMutationsWithoutWAL = new Counter();
278   final Counter dataInMemoryWithoutWAL = new Counter();
279 
280   // Debug why CAS operations are taking a while.
281   final Counter checkAndMutateChecksPassed = new Counter();
282   final Counter checkAndMutateChecksFailed = new Counter();
283 
284   //Number of requests
285   final Counter readRequestsCount = new Counter();
286   final Counter writeRequestsCount = new Counter();
287 
288   // Number of requests blocked by memstore size.
289   private final Counter blockedRequestsCount = new Counter();
290 
291   // Compaction counters
292   final AtomicLong compactionsFinished = new AtomicLong(0L);
293   final AtomicLong compactionNumFilesCompacted = new AtomicLong(0L);
294   final AtomicLong compactionNumBytesCompacted = new AtomicLong(0L);
295 
296   private final WAL wal;
297   private final HRegionFileSystem fs;
298   protected final Configuration conf;
299   private final Configuration baseConf;
300   private final KeyValue.KVComparator comparator;
301   private final int rowLockWaitDuration;
302   static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
303 
304   // The internal wait duration to acquire a lock before read/update
305   // from the region. It is not per row. The purpose of this wait time
306   // is to avoid waiting a long time while the region is busy, so that
307   // we can release the IPC handler soon enough to improve the
308   // availability of the region server. It can be adjusted by
309   // tuning configuration "hbase.busy.wait.duration".
310   final long busyWaitDuration;
311   static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
312 
313   // If updating multiple rows in one call, wait longer,
314   // i.e. waiting for busyWaitDuration * # of rows. However,
315   // we can limit the max multiplier.
316   final int maxBusyWaitMultiplier;
317 
318   // Max busy wait duration. There is no point to wait longer than the RPC
319   // purge timeout, when a RPC call will be terminated by the RPC engine.
320   final long maxBusyWaitDuration;
321 
322   // negative number indicates infinite timeout
323   static final long DEFAULT_ROW_PROCESSOR_TIMEOUT = 60 * 1000L;
324   final ExecutorService rowProcessorExecutor = Executors.newCachedThreadPool();
325 
326   private final ConcurrentHashMap<RegionScanner, Long> scannerReadPoints;
327 
328   /**
329    * The sequence ID that was encountered when this region was opened.
330    */
331   private long openSeqNum = HConstants.NO_SEQNUM;
332 
333   /**
334    * The default setting for whether to enable on-demand CF loading for
335    * scan requests to this region. Requests can override it.
336    */
337   private boolean isLoadingCfsOnDemandDefault = false;
338 
339   private final AtomicInteger majorInProgress = new AtomicInteger(0);
340   private final AtomicInteger minorInProgress = new AtomicInteger(0);
341 
342   //
343   // Context: During replay we want to ensure that we do not lose any data. So, we
344   // have to be conservative in how we replay wals. For each store, we calculate
345   // the maxSeqId up to which the store was flushed. And, skip the edits which
346   // are equal to or lower than maxSeqId for each store.
347   // The following map is populated when opening the region
348   Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
349 
350   /** Saved state from replaying prepare flush cache */
351   private PrepareFlushResult prepareFlushResult = null;
352 
353   /**
354    * Config setting for whether to allow writes when a region is in recovering or not.
355    */
356   private boolean disallowWritesInRecovering = false;
357 
358   // when a region is in recovering state, it can only accept writes not reads
359   private volatile boolean recovering = false;
360 
361   private volatile Optional<ConfigurationManager> configurationManager;
362 
363   /**
364    * @return The smallest mvcc readPoint across all the scanners in this
365    * region. Writes older than this readPoint, are included in every
366    * read operation.
367    */
368   public long getSmallestReadPoint() {
369     long minimumReadPoint;
370     // We need to ensure that while we are calculating the smallestReadPoint
371     // no new RegionScanners can grab a readPoint that we are unaware of.
372     // We achieve this by synchronizing on the scannerReadPoints object.
373     synchronized(scannerReadPoints) {
374       minimumReadPoint = mvcc.getReadPoint();
375 
376       for (Long readPoint: this.scannerReadPoints.values()) {
377         if (readPoint < minimumReadPoint) {
378           minimumReadPoint = readPoint;
379         }
380       }
381     }
382     return minimumReadPoint;
383   }
384 
385   /*
386    * Data structure of write state flags used coordinating flushes,
387    * compactions and closes.
388    */
389   static class WriteState {
390     // Set while a memstore flush is happening.
391     volatile boolean flushing = false;
392     // Set when a flush has been requested.
393     volatile boolean flushRequested = false;
394     // Number of compactions running.
395     AtomicInteger compacting = new AtomicInteger(0);
396     // Gets set in close. If set, cannot compact or flush again.
397     volatile boolean writesEnabled = true;
398     // Set if region is read-only
399     volatile boolean readOnly = false;
400     // whether the reads are enabled. This is different than readOnly, because readOnly is
401     // static in the lifetime of the region, while readsEnabled is dynamic
402     volatile boolean readsEnabled = true;
403 
404     /**
405      * Set flags that make this region read-only.
406      *
407      * @param onOff flip value for region r/o setting
408      */
409     synchronized void setReadOnly(final boolean onOff) {
410       this.writesEnabled = !onOff;
411       this.readOnly = onOff;
412     }
413 
414     boolean isReadOnly() {
415       return this.readOnly;
416     }
417 
418     boolean isFlushRequested() {
419       return this.flushRequested;
420     }
421 
422     void setReadsEnabled(boolean readsEnabled) {
423       this.readsEnabled = readsEnabled;
424     }
425 
426     static final long HEAP_SIZE = ClassSize.align(
427         ClassSize.OBJECT + 5 * Bytes.SIZEOF_BOOLEAN);
428   }
429 
430   /**
431    * Objects from this class are created when flushing to describe all the different states that
432    * that method ends up in. The Result enum describes those states. The sequence id should only
433    * be specified if the flush was successful, and the failure message should only be specified
434    * if it didn't flush.
435    */
436   public static class FlushResultImpl implements FlushResult {
437     final Result result;
438     final String failureReason;
439     final long flushSequenceId;
440     final boolean wroteFlushWalMarker;
441 
442     /**
443      * Convenience constructor to use when the flush is successful, the failure message is set to
444      * null.
445      * @param result Expecting FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_COMPACTION_NEEDED.
446      * @param flushSequenceId Generated sequence id that comes right after the edits in the
447      *                        memstores.
448      */
449     FlushResultImpl(Result result, long flushSequenceId) {
450       this(result, flushSequenceId, null, false);
451       assert result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result
452           .FLUSHED_COMPACTION_NEEDED;
453     }
454 
455     /**
456      * Convenience constructor to use when we cannot flush.
457      * @param result Expecting CANNOT_FLUSH_MEMSTORE_EMPTY or CANNOT_FLUSH.
458      * @param failureReason Reason why we couldn't flush.
459      */
460     FlushResultImpl(Result result, String failureReason, boolean wroteFlushMarker) {
461       this(result, -1, failureReason, wroteFlushMarker);
462       assert result == Result.CANNOT_FLUSH_MEMSTORE_EMPTY || result == Result.CANNOT_FLUSH;
463     }
464 
465     /**
466      * Constructor with all the parameters.
467      * @param result Any of the Result.
468      * @param flushSequenceId Generated sequence id if the memstores were flushed else -1.
469      * @param failureReason Reason why we couldn't flush, or null.
470      */
471     FlushResultImpl(Result result, long flushSequenceId, String failureReason,
472       boolean wroteFlushMarker) {
473       this.result = result;
474       this.flushSequenceId = flushSequenceId;
475       this.failureReason = failureReason;
476       this.wroteFlushWalMarker = wroteFlushMarker;
477     }
478 
479     /**
480      * Convenience method, the equivalent of checking if result is
481      * FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_NO_COMPACTION_NEEDED.
482      * @return true if the memstores were flushed, else false.
483      */
484     @Override
485     public boolean isFlushSucceeded() {
486       return result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result
487           .FLUSHED_COMPACTION_NEEDED;
488     }
489 
490     /**
491      * Convenience method, the equivalent of checking if result is FLUSHED_COMPACTION_NEEDED.
492      * @return True if the flush requested a compaction, else false (doesn't even mean it flushed).
493      */
494     @Override
495     public boolean isCompactionNeeded() {
496       return result == Result.FLUSHED_COMPACTION_NEEDED;
497     }
498 
499     @Override
500     public String toString() {
501       return new StringBuilder()
502         .append("flush result:").append(result).append(", ")
503         .append("failureReason:").append(failureReason).append(",")
504         .append("flush seq id").append(flushSequenceId).toString();
505     }
506 
507     @Override
508     public Result getResult() {
509       return result;
510     }
511   }
512 
513   /** A result object from prepare flush cache stage */
514   @VisibleForTesting
515   static class PrepareFlushResult {
516     final FlushResult result; // indicating a failure result from prepare
517     final TreeMap<byte[], StoreFlushContext> storeFlushCtxs;
518     final TreeMap<byte[], List<Path>> committedFiles;
519     final TreeMap<byte[], Long> storeFlushableSize;
520     final long startTime;
521     final long flushOpSeqId;
522     final long flushedSeqId;
523     final long totalFlushableSize;
524 
525     /** Constructs an early exit case */
526     PrepareFlushResult(FlushResult result, long flushSeqId) {
527       this(result, null, null, null, Math.max(0, flushSeqId), 0, 0, 0);
528     }
529 
530     /** Constructs a successful prepare flush result */
531     PrepareFlushResult(
532       TreeMap<byte[], StoreFlushContext> storeFlushCtxs,
533       TreeMap<byte[], List<Path>> committedFiles,
534       TreeMap<byte[], Long> storeFlushableSize, long startTime, long flushSeqId,
535       long flushedSeqId, long totalFlushableSize) {
536       this(null, storeFlushCtxs, committedFiles, storeFlushableSize, startTime,
537         flushSeqId, flushedSeqId, totalFlushableSize);
538     }
539 
540     private PrepareFlushResult(
541       FlushResult result,
542       TreeMap<byte[], StoreFlushContext> storeFlushCtxs,
543       TreeMap<byte[], List<Path>> committedFiles,
544       TreeMap<byte[], Long> storeFlushableSize, long startTime, long flushSeqId,
545       long flushedSeqId, long totalFlushableSize) {
546       this.result = result;
547       this.storeFlushCtxs = storeFlushCtxs;
548       this.committedFiles = committedFiles;
549       this.storeFlushableSize = storeFlushableSize;
550       this.startTime = startTime;
551       this.flushOpSeqId = flushSeqId;
552       this.flushedSeqId = flushedSeqId;
553       this.totalFlushableSize = totalFlushableSize;
554     }
555 
556     public FlushResult getResult() {
557       return this.result;
558     }
559   }
560 
561   final WriteState writestate = new WriteState();
562 
563   long memstoreFlushSize;
564   final long timestampSlop;
565   final long rowProcessorTimeout;
566 
567   // Last flush time for each Store. Useful when we are flushing for each column
568   private final ConcurrentMap<Store, Long> lastStoreFlushTimeMap =
569       new ConcurrentHashMap<Store, Long>();
570 
571   final RegionServerServices rsServices;
572   private RegionServerAccounting rsAccounting;
573   private long flushCheckInterval;
574   // flushPerChanges is to prevent too many changes in memstore
575   private long flushPerChanges;
576   private long blockingMemStoreSize;
577   final long threadWakeFrequency;
578   // Used to guard closes
579   final ReentrantReadWriteLock lock =
580     new ReentrantReadWriteLock();
581 
582   // Stop updates lock
583   private final ReentrantReadWriteLock updatesLock =
584     new ReentrantReadWriteLock();
585   private boolean splitRequest;
586   private byte[] explicitSplitPoint = null;
587 
588   private final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
589 
590   // Coprocessor host
591   private RegionCoprocessorHost coprocessorHost;
592 
593   private HTableDescriptor htableDescriptor = null;
594   private RegionSplitPolicy splitPolicy;
595   private FlushPolicy flushPolicy;
596 
597   private final MetricsRegion metricsRegion;
598   private final MetricsRegionWrapperImpl metricsRegionWrapper;
599   private final Durability durability;
600   private final boolean regionStatsEnabled;
601 
602   /**
603    * HRegion constructor. This constructor should only be used for testing and
604    * extensions.  Instances of HRegion should be instantiated with the
605    * {@link HRegion#createHRegion} or {@link HRegion#openHRegion} method.
606    *
607    * @param tableDir qualified path of directory where region should be located,
608    * usually the table directory.
609    * @param wal The WAL is the outbound log for any updates to the HRegion
610    * The wal file is a logfile from the previous execution that's
611    * custom-computed for this HRegion. The HRegionServer computes and sorts the
612    * appropriate wal info for this HRegion. If there is a previous wal file
613    * (implying that the HRegion has been written-to before), then read it from
614    * the supplied path.
615    * @param fs is the filesystem.
616    * @param confParam is global configuration settings.
617    * @param regionInfo - HRegionInfo that describes the region
618    * is new), then read them from the supplied path.
619    * @param htd the table descriptor
620    * @param rsServices reference to {@link RegionServerServices} or null
621    * @deprecated Use other constructors.
622    */
623   @Deprecated
624   @VisibleForTesting
625   public HRegion(final Path tableDir, final WAL wal, final FileSystem fs,
626       final Configuration confParam, final HRegionInfo regionInfo,
627       final HTableDescriptor htd, final RegionServerServices rsServices) {
628     this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo),
629       wal, confParam, htd, rsServices);
630   }
631 
632   /**
633    * HRegion constructor. This constructor should only be used for testing and
634    * extensions.  Instances of HRegion should be instantiated with the
635    * {@link HRegion#createHRegion} or {@link HRegion#openHRegion} method.
636    *
637    * @param fs is the filesystem.
638    * @param wal The WAL is the outbound log for any updates to the HRegion
639    * The wal file is a logfile from the previous execution that's
640    * custom-computed for this HRegion. The HRegionServer computes and sorts the
641    * appropriate wal info for this HRegion. If there is a previous wal file
642    * (implying that the HRegion has been written-to before), then read it from
643    * the supplied path.
644    * @param confParam is global configuration settings.
645    * @param htd the table descriptor
646    * @param rsServices reference to {@link RegionServerServices} or null
647    */
648   public HRegion(final HRegionFileSystem fs, final WAL wal, final Configuration confParam,
649       final HTableDescriptor htd, final RegionServerServices rsServices) {
650     if (htd == null) {
651       throw new IllegalArgumentException("Need table descriptor");
652     }
653 
654     if (confParam instanceof CompoundConfiguration) {
655       throw new IllegalArgumentException("Need original base configuration");
656     }
657 
658     this.comparator = fs.getRegionInfo().getComparator();
659     this.wal = wal;
660     this.fs = fs;
661 
662     // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
663     this.baseConf = confParam;
664     this.conf = new CompoundConfiguration()
665       .add(confParam)
666       .addStringMap(htd.getConfiguration())
667       .addWritableMap(htd.getValues());
668     this.flushCheckInterval = conf.getInt(MEMSTORE_PERIODIC_FLUSH_INTERVAL,
669         DEFAULT_CACHE_FLUSH_INTERVAL);
670     this.flushPerChanges = conf.getLong(MEMSTORE_FLUSH_PER_CHANGES, DEFAULT_FLUSH_PER_CHANGES);
671     if (this.flushPerChanges > MAX_FLUSH_PER_CHANGES) {
672       throw new IllegalArgumentException(MEMSTORE_FLUSH_PER_CHANGES + " can not exceed "
673           + MAX_FLUSH_PER_CHANGES);
674     }
675     this.rowLockWaitDuration = conf.getInt("hbase.rowlock.wait.duration",
676                     DEFAULT_ROWLOCK_WAIT_DURATION);
677 
678     this.maxWaitForSeqId = conf.getInt(MAX_WAIT_FOR_SEQ_ID_KEY, DEFAULT_MAX_WAIT_FOR_SEQ_ID);
679     this.isLoadingCfsOnDemandDefault = conf.getBoolean(LOAD_CFS_ON_DEMAND_CONFIG_KEY, true);
680     this.htableDescriptor = htd;
681     this.rsServices = rsServices;
682     this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
683     setHTableSpecificConf();
684     this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
685 
686     this.busyWaitDuration = conf.getLong(
687       "hbase.busy.wait.duration", DEFAULT_BUSY_WAIT_DURATION);
688     this.maxBusyWaitMultiplier = conf.getInt("hbase.busy.wait.multiplier.max", 2);
689     if (busyWaitDuration * maxBusyWaitMultiplier <= 0L) {
690       throw new IllegalArgumentException("Invalid hbase.busy.wait.duration ("
691         + busyWaitDuration + ") or hbase.busy.wait.multiplier.max ("
692         + maxBusyWaitMultiplier + "). Their product should be positive");
693     }
694     this.maxBusyWaitDuration = conf.getLong("hbase.ipc.client.call.purge.timeout",
695       2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
696 
697     /*
698      * timestamp.slop provides a server-side constraint on the timestamp. This
699      * assumes that you base your TS around currentTimeMillis(). In this case,
700      * throw an error to the user if the user-specified TS is newer than now +
701      * slop. LATEST_TIMESTAMP == don't use this functionality
702      */
703     this.timestampSlop = conf.getLong(
704         "hbase.hregion.keyvalue.timestamp.slop.millisecs",
705         HConstants.LATEST_TIMESTAMP);
706 
707     /**
708      * Timeout for the process time in processRowsWithLocks().
709      * Use -1 to switch off time bound.
710      */
711     this.rowProcessorTimeout = conf.getLong(
712         "hbase.hregion.row.processor.timeout", DEFAULT_ROW_PROCESSOR_TIMEOUT);
713     this.durability = htd.getDurability() == Durability.USE_DEFAULT
714         ? DEFAULT_DURABILITY
715         : htd.getDurability();
716     if (rsServices != null) {
717       this.rsAccounting = this.rsServices.getRegionServerAccounting();
718       // don't initialize coprocessors if not running within a regionserver
719       // TODO: revisit if coprocessors should load in other cases
720       this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
721       this.metricsRegionWrapper = new MetricsRegionWrapperImpl(this);
722       this.metricsRegion = new MetricsRegion(this.metricsRegionWrapper);
723 
724       Map<String, Region> recoveringRegions = rsServices.getRecoveringRegions();
725       String encodedName = getRegionInfo().getEncodedName();
726       if (recoveringRegions != null && recoveringRegions.containsKey(encodedName)) {
727         this.recovering = true;
728         recoveringRegions.put(encodedName, this);
729       }
730     } else {
731       this.metricsRegionWrapper = null;
732       this.metricsRegion = null;
733     }
734     if (LOG.isDebugEnabled()) {
735       // Write out region name as string and its encoded name.
736       LOG.debug("Instantiated " + this);
737     }
738 
739     // by default, we allow writes against a region when it's in recovering
740     this.disallowWritesInRecovering =
741         conf.getBoolean(HConstants.DISALLOW_WRITES_IN_RECOVERING,
742           HConstants.DEFAULT_DISALLOW_WRITES_IN_RECOVERING_CONFIG);
743     configurationManager = Optional.absent();
744 
745     // disable stats tracking system tables, but check the config for everything else
746     this.regionStatsEnabled = htd.getTableName().getNamespaceAsString().equals(
747         NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR) ?
748           false :
749           conf.getBoolean(HConstants.ENABLE_CLIENT_BACKPRESSURE,
750               HConstants.DEFAULT_ENABLE_CLIENT_BACKPRESSURE);
751   }
752 
753   void setHTableSpecificConf() {
754     if (this.htableDescriptor == null) return;
755     long flushSize = this.htableDescriptor.getMemStoreFlushSize();
756 
757     if (flushSize <= 0) {
758       flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
759         HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
760     }
761     this.memstoreFlushSize = flushSize;
762     this.blockingMemStoreSize = this.memstoreFlushSize *
763         conf.getLong(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER,
764                 HConstants.DEFAULT_HREGION_MEMSTORE_BLOCK_MULTIPLIER);
765   }
766 
767   /**
768    * Initialize this region.
769    * Used only by tests and SplitTransaction to reopen the region.
770    * You should use createHRegion() or openHRegion()
771    * @return What the next sequence (edit) id should be.
772    * @throws IOException e
773    * @deprecated use HRegion.createHRegion() or HRegion.openHRegion()
774    */
775   @Deprecated
776   public long initialize() throws IOException {
777     return initialize(null);
778   }
779 
780   /**
781    * Initialize this region.
782    *
783    * @param reporter Tickle every so often if initialize is taking a while.
784    * @return What the next sequence (edit) id should be.
785    * @throws IOException e
786    */
787   private long initialize(final CancelableProgressable reporter) throws IOException {
788 
789     //Refuse to open the region if there is no column family in the table
790     if (htableDescriptor.getColumnFamilies().length == 0) {
791       throw new DoNotRetryIOException("Table " + htableDescriptor.getNameAsString() +
792           " should have at least one column family.");
793     }
794 
795     MonitoredTask status = TaskMonitor.get().createStatus("Initializing region " + this);
796     long nextSeqId = -1;
797     try {
798       nextSeqId = initializeRegionInternals(reporter, status);
799       return nextSeqId;
800     } finally {
801       // nextSeqid will be -1 if the initialization fails.
802       // At least it will be 0 otherwise.
803       if (nextSeqId == -1) {
804         status.abort("Exception during region " + getRegionInfo().getRegionNameAsString() +
805           " initialization.");
806       }
807     }
808   }
809 
810   private long initializeRegionInternals(final CancelableProgressable reporter,
811       final MonitoredTask status) throws IOException {
812     if (coprocessorHost != null) {
813       status.setStatus("Running coprocessor pre-open hook");
814       coprocessorHost.preOpen();
815     }
816 
817     // Write HRI to a file in case we need to recover hbase:meta
818     status.setStatus("Writing region info on filesystem");
819     fs.checkRegionInfoOnFilesystem();
820 
821     // Initialize all the HStores
822     status.setStatus("Initializing all the Stores");
823     long maxSeqId = initializeStores(reporter, status);
824     this.mvcc.advanceTo(maxSeqId);
825     if (ServerRegionReplicaUtil.shouldReplayRecoveredEdits(this)) {
826       // Recover any edits if available.
827       maxSeqId = Math.max(maxSeqId,
828         replayRecoveredEditsIfAny(this.fs.getRegionDir(), maxSeqIdInStores, reporter, status));
829       // Make sure mvcc is up to max.
830       this.mvcc.advanceTo(maxSeqId);
831     }
832     this.lastReplayedOpenRegionSeqId = maxSeqId;
833 
834     this.writestate.setReadOnly(ServerRegionReplicaUtil.isReadOnly(this));
835     this.writestate.flushRequested = false;
836     this.writestate.compacting.set(0);
837 
838     if (this.writestate.writesEnabled) {
839       // Remove temporary data left over from old regions
840       status.setStatus("Cleaning up temporary data from old regions");
841       fs.cleanupTempDir();
842     }
843 
844     if (this.writestate.writesEnabled) {
845       status.setStatus("Cleaning up detritus from prior splits");
846       // Get rid of any splits or merges that were lost in-progress.  Clean out
847       // these directories here on open.  We may be opening a region that was
848       // being split but we crashed in the middle of it all.
849       fs.cleanupAnySplitDetritus();
850       fs.cleanupMergesDir();
851     }
852 
853     // Initialize split policy
854     this.splitPolicy = RegionSplitPolicy.create(this, conf);
855 
856     // Initialize flush policy
857     this.flushPolicy = FlushPolicyFactory.create(this, conf);
858 
859     long lastFlushTime = EnvironmentEdgeManager.currentTime();
860     for (Store store: stores.values()) {
861       this.lastStoreFlushTimeMap.put(store, lastFlushTime);
862     }
863 
864     // Use maximum of log sequenceid or that which was found in stores
865     // (particularly if no recovered edits, seqid will be -1).
866     long nextSeqid = maxSeqId;
867 
868     // In distributedLogReplay mode, we don't know the last change sequence number because region
869     // is opened before recovery completes. So we add a safety bumper to avoid new sequence number
870     // overlaps used sequence numbers
871     if (this.writestate.writesEnabled) {
872       nextSeqid = WALSplitter.writeRegionSequenceIdFile(this.fs.getFileSystem(), this.fs
873           .getRegionDir(), nextSeqid, (this.recovering ? (this.flushPerChanges + 10000000) : 1));
874     } else {
875       nextSeqid++;
876     }
877 
878     LOG.info("Onlined " + this.getRegionInfo().getShortNameToLog() +
879       "; next sequenceid=" + nextSeqid);
880 
881     // A region can be reopened if failed a split; reset flags
882     this.closing.set(false);
883     this.closed.set(false);
884 
885     if (coprocessorHost != null) {
886       status.setStatus("Running coprocessor post-open hooks");
887       coprocessorHost.postOpen();
888     }
889 
890     status.markComplete("Region opened successfully");
891     return nextSeqid;
892   }
893 
894   /**
895    * Open all Stores.
896    * @param reporter
897    * @param status
898    * @return Highest sequenceId found out in a Store.
899    * @throws IOException
900    */
901   private long initializeStores(final CancelableProgressable reporter, MonitoredTask status)
902   throws IOException {
903     // Load in all the HStores.
904 
905     long maxSeqId = -1;
906     // initialized to -1 so that we pick up MemstoreTS from column families
907     long maxMemstoreTS = -1;
908 
909     if (!htableDescriptor.getFamilies().isEmpty()) {
910       // initialize the thread pool for opening stores in parallel.
911       ThreadPoolExecutor storeOpenerThreadPool =
912         getStoreOpenAndCloseThreadPool("StoreOpener-" + this.getRegionInfo().getShortNameToLog());
913       CompletionService<HStore> completionService =
914         new ExecutorCompletionService<HStore>(storeOpenerThreadPool);
915 
916       // initialize each store in parallel
917       for (final HColumnDescriptor family : htableDescriptor.getFamilies()) {
918         status.setStatus("Instantiating store for column family " + family);
919         completionService.submit(new Callable<HStore>() {
920           @Override
921           public HStore call() throws IOException {
922             return instantiateHStore(family);
923           }
924         });
925       }
926       boolean allStoresOpened = false;
927       try {
928         for (int i = 0; i < htableDescriptor.getFamilies().size(); i++) {
929           Future<HStore> future = completionService.take();
930           HStore store = future.get();
931           this.stores.put(store.getFamily().getName(), store);
932 
933           long storeMaxSequenceId = store.getMaxSequenceId();
934           maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
935               storeMaxSequenceId);
936           if (maxSeqId == -1 || storeMaxSequenceId > maxSeqId) {
937             maxSeqId = storeMaxSequenceId;
938           }
939           long maxStoreMemstoreTS = store.getMaxMemstoreTS();
940           if (maxStoreMemstoreTS > maxMemstoreTS) {
941             maxMemstoreTS = maxStoreMemstoreTS;
942           }
943         }
944         allStoresOpened = true;
945       } catch (InterruptedException e) {
946         throw (InterruptedIOException)new InterruptedIOException().initCause(e);
947       } catch (ExecutionException e) {
948         throw new IOException(e.getCause());
949       } finally {
950         storeOpenerThreadPool.shutdownNow();
951         if (!allStoresOpened) {
952           // something went wrong, close all opened stores
953           LOG.error("Could not initialize all stores for the region=" + this);
954           for (Store store : this.stores.values()) {
955             try {
956               store.close();
957             } catch (IOException e) {
958               LOG.warn(e.getMessage());
959             }
960           }
961         }
962       }
963     }
964     return Math.max(maxSeqId, maxMemstoreTS + 1);
965   }
966 
967   private void initializeWarmup(final CancelableProgressable reporter) throws IOException {
968     MonitoredTask status = TaskMonitor.get().createStatus("Initializing region " + this);
969 
970     // Initialize all the HStores
971     status.setStatus("Warming up all the Stores");
972     initializeStores(reporter, status);
973   }
974 
975   /**
976    * @return Map of StoreFiles by column family
977    */
978   private NavigableMap<byte[], List<Path>> getStoreFiles() {
979     NavigableMap<byte[], List<Path>> allStoreFiles =
980       new TreeMap<byte[], List<Path>>(Bytes.BYTES_COMPARATOR);
981     for (Store store: getStores()) {
982       Collection<StoreFile> storeFiles = store.getStorefiles();
983       if (storeFiles == null) continue;
984       List<Path> storeFileNames = new ArrayList<Path>();
985       for (StoreFile storeFile: storeFiles) {
986         storeFileNames.add(storeFile.getPath());
987       }
988       allStoreFiles.put(store.getFamily().getName(), storeFileNames);
989     }
990     return allStoreFiles;
991   }
992 
993   private void writeRegionOpenMarker(WAL wal, long openSeqId) throws IOException {
994     Map<byte[], List<Path>> storeFiles = getStoreFiles();
995     RegionEventDescriptor regionOpenDesc = ProtobufUtil.toRegionEventDescriptor(
996       RegionEventDescriptor.EventType.REGION_OPEN, getRegionInfo(), openSeqId,
997       getRegionServerServices().getServerName(), storeFiles);
998     WALUtil.writeRegionEventMarker(wal, getTableDesc(), getRegionInfo(), regionOpenDesc, mvcc);
999   }
1000 
1001   private void writeRegionCloseMarker(WAL wal) throws IOException {
1002     Map<byte[], List<Path>> storeFiles = getStoreFiles();
1003     RegionEventDescriptor regionEventDesc = ProtobufUtil.toRegionEventDescriptor(
1004       RegionEventDescriptor.EventType.REGION_CLOSE, getRegionInfo(), mvcc.getReadPoint(),
1005       getRegionServerServices().getServerName(), storeFiles);
1006     WALUtil.writeRegionEventMarker(wal, getTableDesc(), getRegionInfo(), regionEventDesc, mvcc);
1007 
1008     // Store SeqId in HDFS when a region closes
1009     // checking region folder exists is due to many tests which delete the table folder while a
1010     // table is still online
1011     if (this.fs.getFileSystem().exists(this.fs.getRegionDir())) {
1012       WALSplitter.writeRegionSequenceIdFile(this.fs.getFileSystem(), this.fs.getRegionDir(),
1013         mvcc.getReadPoint(), 0);
1014     }
1015   }
1016 
1017   /**
1018    * @return True if this region has references.
1019    */
1020   public boolean hasReferences() {
1021     for (Store store : this.stores.values()) {
1022       if (store.hasReferences()) return true;
1023     }
1024     return false;
1025   }
1026 
1027   @Override
1028   public HDFSBlocksDistribution getHDFSBlocksDistribution() {
1029     HDFSBlocksDistribution hdfsBlocksDistribution =
1030       new HDFSBlocksDistribution();
1031     synchronized (this.stores) {
1032       for (Store store : this.stores.values()) {
1033         Collection<StoreFile> storeFiles = store.getStorefiles();
1034         if (storeFiles == null) continue;
1035         for (StoreFile sf : storeFiles) {
1036           HDFSBlocksDistribution storeFileBlocksDistribution =
1037             sf.getHDFSBlockDistribution();
1038           hdfsBlocksDistribution.add(storeFileBlocksDistribution);
1039         }
1040       }
1041     }
1042     return hdfsBlocksDistribution;
1043   }
1044 
1045   /**
1046    * This is a helper function to compute HDFS block distribution on demand
1047    * @param conf configuration
1048    * @param tableDescriptor HTableDescriptor of the table
1049    * @param regionInfo encoded name of the region
1050    * @return The HDFS blocks distribution for the given region.
1051    * @throws IOException
1052    */
1053   public static HDFSBlocksDistribution computeHDFSBlocksDistribution(final Configuration conf,
1054       final HTableDescriptor tableDescriptor, final HRegionInfo regionInfo) throws IOException {
1055     Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), tableDescriptor.getTableName());
1056     return computeHDFSBlocksDistribution(conf, tableDescriptor, regionInfo, tablePath);
1057   }
1058 
1059   /**
1060    * This is a helper function to compute HDFS block distribution on demand
1061    * @param conf configuration
1062    * @param tableDescriptor HTableDescriptor of the table
1063    * @param regionInfo encoded name of the region
1064    * @param tablePath the table directory
1065    * @return The HDFS blocks distribution for the given region.
1066    * @throws IOException
1067    */
1068   public static HDFSBlocksDistribution computeHDFSBlocksDistribution(final Configuration conf,
1069       final HTableDescriptor tableDescriptor, final HRegionInfo regionInfo,  Path tablePath)
1070       throws IOException {
1071     HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
1072     FileSystem fs = tablePath.getFileSystem(conf);
1073 
1074     HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tablePath, regionInfo);
1075     for (HColumnDescriptor family: tableDescriptor.getFamilies()) {
1076       Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family.getNameAsString());
1077       if (storeFiles == null) continue;
1078       for (StoreFileInfo storeFileInfo : storeFiles) {
1079         try {
1080           hdfsBlocksDistribution.add(storeFileInfo.computeHDFSBlocksDistribution(fs));
1081         } catch (IOException ioe) {
1082           LOG.warn("Error getting hdfs block distribution for " + storeFileInfo);
1083         }
1084       }
1085     }
1086     return hdfsBlocksDistribution;
1087   }
1088 
1089   /**
1090    * Increase the size of mem store in this region and the size of global mem
1091    * store
1092    * @return the size of memstore in this region
1093    */
1094   public long addAndGetGlobalMemstoreSize(long memStoreSize) {
1095     if (this.rsAccounting != null) {
1096       rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize);
1097     }
1098     return this.memstoreSize.addAndGet(memStoreSize);
1099   }
1100 
1101   @Override
1102   public HRegionInfo getRegionInfo() {
1103     return this.fs.getRegionInfo();
1104   }
1105 
1106   /**
1107    * @return Instance of {@link RegionServerServices} used by this HRegion.
1108    * Can be null.
1109    */
1110   RegionServerServices getRegionServerServices() {
1111     return this.rsServices;
1112   }
1113 
1114   @Override
1115   public long getReadRequestsCount() {
1116     return readRequestsCount.get();
1117   }
1118 
1119   @Override
1120   public void updateReadRequestsCount(long i) {
1121     readRequestsCount.add(i);
1122   }
1123 
1124   @Override
1125   public long getWriteRequestsCount() {
1126     return writeRequestsCount.get();
1127   }
1128 
1129   @Override
1130   public void updateWriteRequestsCount(long i) {
1131     writeRequestsCount.add(i);
1132   }
1133 
1134   @Override
1135   public long getMemstoreSize() {
1136     return memstoreSize.get();
1137   }
1138 
1139   @Override
1140   public long getNumMutationsWithoutWAL() {
1141     return numMutationsWithoutWAL.get();
1142   }
1143 
1144   @Override
1145   public long getDataInMemoryWithoutWAL() {
1146     return dataInMemoryWithoutWAL.get();
1147   }
1148 
1149   @Override
1150   public long getBlockedRequestsCount() {
1151     return blockedRequestsCount.get();
1152   }
1153 
1154   @Override
1155   public long getCheckAndMutateChecksPassed() {
1156     return checkAndMutateChecksPassed.get();
1157   }
1158 
1159   @Override
1160   public long getCheckAndMutateChecksFailed() {
1161     return checkAndMutateChecksFailed.get();
1162   }
1163 
1164   @Override
1165   public MetricsRegion getMetrics() {
1166     return metricsRegion;
1167   }
1168 
1169   @Override
1170   public boolean isClosed() {
1171     return this.closed.get();
1172   }
1173 
1174   @Override
1175   public boolean isClosing() {
1176     return this.closing.get();
1177   }
1178 
1179   @Override
1180   public boolean isReadOnly() {
1181     return this.writestate.isReadOnly();
1182   }
1183 
1184   /**
1185    * Reset recovering state of current region
1186    */
1187   public void setRecovering(boolean newState) {
1188     boolean wasRecovering = this.recovering;
1189     // before we flip the recovering switch (enabling reads) we should write the region open
1190     // event to WAL if needed
1191     if (wal != null && getRegionServerServices() != null && !writestate.readOnly
1192         && wasRecovering && !newState) {
1193 
1194       // force a flush only if region replication is set up for this region. Otherwise no need.
1195       boolean forceFlush = getTableDesc().getRegionReplication() > 1;
1196 
1197       // force a flush first
1198       MonitoredTask status = TaskMonitor.get().createStatus(
1199         "Flushing region " + this + " because recovery is finished");
1200       try {
1201         if (forceFlush) {
1202           internalFlushcache(status);
1203         }
1204 
1205         status.setStatus("Writing region open event marker to WAL because recovery is finished");
1206         try {
1207           long seqId = openSeqNum;
1208           // obtain a new seqId because we possibly have writes and flushes on top of openSeqNum
1209           if (wal != null) {
1210             seqId = getNextSequenceId(wal);
1211           }
1212           writeRegionOpenMarker(wal, seqId);
1213         } catch (IOException e) {
1214           // We cannot rethrow this exception since we are being called from the zk thread. The
1215           // region has already opened. In this case we log the error, but continue
1216           LOG.warn(getRegionInfo().getEncodedName() + " : was not able to write region opening "
1217               + "event to WAL, continueing", e);
1218         }
1219       } catch (IOException ioe) {
1220         // Distributed log replay semantics does not necessarily require a flush, since the replayed
1221         // data is already written again in the WAL. So failed flush should be fine.
1222         LOG.warn(getRegionInfo().getEncodedName() + " : was not able to flush "
1223             + "event to WAL, continueing", ioe);
1224       } finally {
1225         status.cleanup();
1226       }
1227     }
1228 
1229     this.recovering = newState;
1230     if (wasRecovering && !recovering) {
1231       // Call only when wal replay is over.
1232       coprocessorHost.postLogReplay();
1233     }
1234   }
1235 
1236   @Override
1237   public boolean isRecovering() {
1238     return this.recovering;
1239   }
1240 
1241   @Override
1242   public boolean isAvailable() {
1243     return !isClosed() && !isClosing();
1244   }
1245 
1246   /** @return true if region is splittable */
1247   public boolean isSplittable() {
1248     return isAvailable() && !hasReferences();
1249   }
1250 
1251   /**
1252    * @return true if region is mergeable
1253    */
1254   public boolean isMergeable() {
1255     if (!isAvailable()) {
1256       LOG.debug("Region " + getRegionInfo().getRegionNameAsString()
1257           + " is not mergeable because it is closing or closed");
1258       return false;
1259     }
1260     if (hasReferences()) {
1261       LOG.debug("Region " + getRegionInfo().getRegionNameAsString()
1262           + " is not mergeable because it has references");
1263       return false;
1264     }
1265 
1266     return true;
1267   }
1268 
1269   public boolean areWritesEnabled() {
1270     synchronized(this.writestate) {
1271       return this.writestate.writesEnabled;
1272     }
1273   }
1274 
1275    public MultiVersionConcurrencyControl getMVCC() {
1276      return mvcc;
1277    }
1278 
1279    @Override
1280    public long getMaxFlushedSeqId() {
1281      return maxFlushedSeqId;
1282    }
1283 
1284    @Override
1285    public long getReadpoint(IsolationLevel isolationLevel) {
1286      if (isolationLevel == IsolationLevel.READ_UNCOMMITTED) {
1287        // This scan can read even uncommitted transactions
1288        return Long.MAX_VALUE;
1289      }
1290      return mvcc.getReadPoint();
1291    }
1292 
1293    @Override
1294    public boolean isLoadingCfsOnDemandDefault() {
1295      return this.isLoadingCfsOnDemandDefault;
1296    }
1297 
1298   /**
1299    * Close down this HRegion.  Flush the cache, shut down each HStore, don't
1300    * service any more calls.
1301    *
1302    * <p>This method could take some time to execute, so don't call it from a
1303    * time-sensitive thread.
1304    *
1305    * @return Vector of all the storage files that the HRegion's component
1306    * HStores make use of.  It's a list of all HStoreFile objects. Returns empty
1307    * vector if already closed and null if judged that it should not close.
1308    *
1309    * @throws IOException e
1310    * @throws DroppedSnapshotException Thrown when replay of wal is required
1311    * because a Snapshot was not properly persisted. The region is put in closing mode, and the
1312    * caller MUST abort after this.
1313    */
1314   public Map<byte[], List<StoreFile>> close() throws IOException {
1315     return close(false);
1316   }
1317 
1318   private final Object closeLock = new Object();
1319 
1320   /** Conf key for the periodic flush interval */
1321   public static final String MEMSTORE_PERIODIC_FLUSH_INTERVAL =
1322       "hbase.regionserver.optionalcacheflushinterval";
1323   /** Default interval for the memstore flush */
1324   public static final int DEFAULT_CACHE_FLUSH_INTERVAL = 3600000;
1325   /** Default interval for System tables memstore flush */
1326   public static final int SYSTEM_CACHE_FLUSH_INTERVAL = 300000; // 5 minutes
1327 
1328   /** Conf key to force a flush if there are already enough changes for one region in memstore */
1329   public static final String MEMSTORE_FLUSH_PER_CHANGES =
1330       "hbase.regionserver.flush.per.changes";
1331   public static final long DEFAULT_FLUSH_PER_CHANGES = 30000000; // 30 millions
1332   /**
1333    * The following MAX_FLUSH_PER_CHANGES is large enough because each KeyValue has 20+ bytes
1334    * overhead. Therefore, even 1G empty KVs occupy at least 20GB memstore size for a single region
1335    */
1336   public static final long MAX_FLUSH_PER_CHANGES = 1000000000; // 1G
1337 
1338   /**
1339    * Close down this HRegion.  Flush the cache unless abort parameter is true,
1340    * Shut down each HStore, don't service any more calls.
1341    *
1342    * This method could take some time to execute, so don't call it from a
1343    * time-sensitive thread.
1344    *
1345    * @param abort true if server is aborting (only during testing)
1346    * @return Vector of all the storage files that the HRegion's component
1347    * HStores make use of.  It's a list of HStoreFile objects.  Can be null if
1348    * we are not to close at this time or we are already closed.
1349    *
1350    * @throws IOException e
1351    * @throws DroppedSnapshotException Thrown when replay of wal is required
1352    * because a Snapshot was not properly persisted. The region is put in closing mode, and the
1353    * caller MUST abort after this.
1354    */
1355   public Map<byte[], List<StoreFile>> close(final boolean abort) throws IOException {
1356     // Only allow one thread to close at a time. Serialize them so dual
1357     // threads attempting to close will run up against each other.
1358     MonitoredTask status = TaskMonitor.get().createStatus(
1359         "Closing region " + this +
1360         (abort ? " due to abort" : ""));
1361 
1362     status.setStatus("Waiting for close lock");
1363     try {
1364       synchronized (closeLock) {
1365         return doClose(abort, status);
1366       }
1367     } finally {
1368       status.cleanup();
1369     }
1370   }
1371 
1372   /**
1373    * Exposed for some very specific unit tests.
1374    */
1375   @VisibleForTesting
1376   public void setClosing(boolean closing) {
1377     this.closing.set(closing);
1378   }
1379 
1380   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK_EXCEPTION_PATH",
1381       justification="I think FindBugs is confused")
1382   private Map<byte[], List<StoreFile>> doClose(final boolean abort, MonitoredTask status)
1383       throws IOException {
1384     if (isClosed()) {
1385       LOG.warn("Region " + this + " already closed");
1386       return null;
1387     }
1388 
1389     if (coprocessorHost != null) {
1390       status.setStatus("Running coprocessor pre-close hooks");
1391       this.coprocessorHost.preClose(abort);
1392     }
1393 
1394     status.setStatus("Disabling compacts and flushes for region");
1395     boolean canFlush = true;
1396     synchronized (writestate) {
1397       // Disable compacting and flushing by background threads for this
1398       // region.
1399       canFlush = !writestate.readOnly;
1400       writestate.writesEnabled = false;
1401       LOG.debug("Closing " + this + ": disabling compactions & flushes");
1402       waitForFlushesAndCompactions();
1403     }
1404     // If we were not just flushing, is it worth doing a preflush...one
1405     // that will clear out of the bulk of the memstore before we put up
1406     // the close flag?
1407     if (!abort && worthPreFlushing() && canFlush) {
1408       status.setStatus("Pre-flushing region before close");
1409       LOG.info("Running close preflush of " + getRegionInfo().getRegionNameAsString());
1410       try {
1411         internalFlushcache(status);
1412       } catch (IOException ioe) {
1413         // Failed to flush the region. Keep going.
1414         status.setStatus("Failed pre-flush " + this + "; " + ioe.getMessage());
1415       }
1416     }
1417 
1418     // block waiting for the lock for closing
1419     lock.writeLock().lock(); // FindBugs: Complains UL_UNRELEASED_LOCK_EXCEPTION_PATH but seems fine
1420     this.closing.set(true);
1421     status.setStatus("Disabling writes for close");
1422     try {
1423       if (this.isClosed()) {
1424         status.abort("Already got closed by another process");
1425         // SplitTransaction handles the null
1426         return null;
1427       }
1428       LOG.debug("Updates disabled for region " + this);
1429       // Don't flush the cache if we are aborting
1430       if (!abort && canFlush) {
1431         int flushCount = 0;
1432         while (this.memstoreSize.get() > 0) {
1433           try {
1434             if (flushCount++ > 0) {
1435               int actualFlushes = flushCount - 1;
1436               if (actualFlushes > 5) {
1437                 // If we tried 5 times and are unable to clear memory, abort
1438                 // so we do not lose data
1439                 throw new DroppedSnapshotException("Failed clearing memory after " +
1440                   actualFlushes + " attempts on region: " +
1441                     Bytes.toStringBinary(getRegionInfo().getRegionName()));
1442               }
1443               LOG.info("Running extra flush, " + actualFlushes +
1444                 " (carrying snapshot?) " + this);
1445             }
1446             internalFlushcache(status);
1447           } catch (IOException ioe) {
1448             status.setStatus("Failed flush " + this + ", putting online again");
1449             synchronized (writestate) {
1450               writestate.writesEnabled = true;
1451             }
1452             // Have to throw to upper layers.  I can't abort server from here.
1453             throw ioe;
1454           }
1455         }
1456       }
1457 
1458       Map<byte[], List<StoreFile>> result =
1459         new TreeMap<byte[], List<StoreFile>>(Bytes.BYTES_COMPARATOR);
1460       if (!stores.isEmpty()) {
1461         // initialize the thread pool for closing stores in parallel.
1462         ThreadPoolExecutor storeCloserThreadPool =
1463           getStoreOpenAndCloseThreadPool("StoreCloserThread-" +
1464             getRegionInfo().getRegionNameAsString());
1465         CompletionService<Pair<byte[], Collection<StoreFile>>> completionService =
1466           new ExecutorCompletionService<Pair<byte[], Collection<StoreFile>>>(storeCloserThreadPool);
1467 
1468         // close each store in parallel
1469         for (final Store store : stores.values()) {
1470           long flushableSize = store.getFlushableSize();
1471           if (!(abort || flushableSize == 0 || writestate.readOnly)) {
1472             if (getRegionServerServices() != null) {
1473               getRegionServerServices().abort("Assertion failed while closing store "
1474                 + getRegionInfo().getRegionNameAsString() + " " + store
1475                 + ". flushableSize expected=0, actual= " + flushableSize
1476                 + ". Current memstoreSize=" + getMemstoreSize() + ". Maybe a coprocessor "
1477                 + "operation failed and left the memstore in a partially updated state.", null);
1478             }
1479           }
1480           completionService
1481               .submit(new Callable<Pair<byte[], Collection<StoreFile>>>() {
1482                 @Override
1483                 public Pair<byte[], Collection<StoreFile>> call() throws IOException {
1484                   return new Pair<byte[], Collection<StoreFile>>(
1485                     store.getFamily().getName(), store.close());
1486                 }
1487               });
1488         }
1489         try {
1490           for (int i = 0; i < stores.size(); i++) {
1491             Future<Pair<byte[], Collection<StoreFile>>> future = completionService.take();
1492             Pair<byte[], Collection<StoreFile>> storeFiles = future.get();
1493             List<StoreFile> familyFiles = result.get(storeFiles.getFirst());
1494             if (familyFiles == null) {
1495               familyFiles = new ArrayList<StoreFile>();
1496               result.put(storeFiles.getFirst(), familyFiles);
1497             }
1498             familyFiles.addAll(storeFiles.getSecond());
1499           }
1500         } catch (InterruptedException e) {
1501           throw (InterruptedIOException)new InterruptedIOException().initCause(e);
1502         } catch (ExecutionException e) {
1503           throw new IOException(e.getCause());
1504         } finally {
1505           storeCloserThreadPool.shutdownNow();
1506         }
1507       }
1508 
1509       status.setStatus("Writing region close event to WAL");
1510       if (!abort && wal != null && getRegionServerServices() != null && !writestate.readOnly) {
1511         writeRegionCloseMarker(wal);
1512       }
1513 
1514       this.closed.set(true);
1515       if (!canFlush) {
1516         addAndGetGlobalMemstoreSize(-memstoreSize.get());
1517       } else if (memstoreSize.get() != 0) {
1518         LOG.error("Memstore size is " + memstoreSize.get());
1519       }
1520       if (coprocessorHost != null) {
1521         status.setStatus("Running coprocessor post-close hooks");
1522         this.coprocessorHost.postClose(abort);
1523       }
1524       if (this.metricsRegion != null) {
1525         this.metricsRegion.close();
1526       }
1527       if (this.metricsRegionWrapper != null) {
1528         Closeables.closeQuietly(this.metricsRegionWrapper);
1529       }
1530       status.markComplete("Closed");
1531       LOG.info("Closed " + this);
1532       return result;
1533     } finally {
1534       lock.writeLock().unlock();
1535     }
1536   }
1537 
1538   @Override
1539   public void waitForFlushesAndCompactions() {
1540     synchronized (writestate) {
1541       if (this.writestate.readOnly) {
1542         // we should not wait for replayed flushed if we are read only (for example in case the
1543         // region is a secondary replica).
1544         return;
1545       }
1546       boolean interrupted = false;
1547       try {
1548         while (writestate.compacting.get() > 0 || writestate.flushing) {
1549           LOG.debug("waiting for " + writestate.compacting + " compactions"
1550             + (writestate.flushing ? " & cache flush" : "") + " to complete for region " + this);
1551           try {
1552             writestate.wait();
1553           } catch (InterruptedException iex) {
1554             // essentially ignore and propagate the interrupt back up
1555             LOG.warn("Interrupted while waiting");
1556             interrupted = true;
1557           }
1558         }
1559       } finally {
1560         if (interrupted) {
1561           Thread.currentThread().interrupt();
1562         }
1563       }
1564     }
1565   }
1566 
1567   protected ThreadPoolExecutor getStoreOpenAndCloseThreadPool(
1568       final String threadNamePrefix) {
1569     int numStores = Math.max(1, this.htableDescriptor.getFamilies().size());
1570     int maxThreads = Math.min(numStores,
1571         conf.getInt(HConstants.HSTORE_OPEN_AND_CLOSE_THREADS_MAX,
1572             HConstants.DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX));
1573     return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);
1574   }
1575 
1576   protected ThreadPoolExecutor getStoreFileOpenAndCloseThreadPool(
1577       final String threadNamePrefix) {
1578     int numStores = Math.max(1, this.htableDescriptor.getFamilies().size());
1579     int maxThreads = Math.max(1,
1580         conf.getInt(HConstants.HSTORE_OPEN_AND_CLOSE_THREADS_MAX,
1581             HConstants.DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX)
1582             / numStores);
1583     return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);
1584   }
1585 
1586   static ThreadPoolExecutor getOpenAndCloseThreadPool(int maxThreads,
1587       final String threadNamePrefix) {
1588     return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
1589       new ThreadFactory() {
1590         private int count = 1;
1591 
1592         @Override
1593         public Thread newThread(Runnable r) {
1594           return new Thread(r, threadNamePrefix + "-" + count++);
1595         }
1596       });
1597   }
1598 
1599    /**
1600     * @return True if its worth doing a flush before we put up the close flag.
1601     */
1602   private boolean worthPreFlushing() {
1603     return this.memstoreSize.get() >
1604       this.conf.getLong("hbase.hregion.preclose.flush.size", 1024 * 1024 * 5);
1605   }
1606 
1607   //////////////////////////////////////////////////////////////////////////////
1608   // HRegion accessors
1609   //////////////////////////////////////////////////////////////////////////////
1610 
1611   @Override
1612   public HTableDescriptor getTableDesc() {
1613     return this.htableDescriptor;
1614   }
1615 
1616   /** @return WAL in use for this region */
1617   public WAL getWAL() {
1618     return this.wal;
1619   }
1620 
1621   /**
1622    * A split takes the config from the parent region & passes it to the daughter
1623    * region's constructor. If 'conf' was passed, you would end up using the HTD
1624    * of the parent region in addition to the new daughter HTD. Pass 'baseConf'
1625    * to the daughter regions to avoid this tricky dedupe problem.
1626    * @return Configuration object
1627    */
1628   Configuration getBaseConf() {
1629     return this.baseConf;
1630   }
1631 
1632   /** @return {@link FileSystem} being used by this region */
1633   public FileSystem getFilesystem() {
1634     return fs.getFileSystem();
1635   }
1636 
1637   /** @return the {@link HRegionFileSystem} used by this region */
1638   public HRegionFileSystem getRegionFileSystem() {
1639     return this.fs;
1640   }
1641 
1642   @Override
1643   public long getEarliestFlushTimeForAllStores() {
1644     return lastStoreFlushTimeMap.isEmpty() ? Long.MAX_VALUE : Collections.min(lastStoreFlushTimeMap
1645         .values());
1646   }
1647 
1648   @Override
1649   public long getOldestHfileTs(boolean majorCompactioOnly) throws IOException {
1650     long result = Long.MAX_VALUE;
1651     for (Store store : getStores()) {
1652       Collection<StoreFile> storeFiles = store.getStorefiles();
1653       if (storeFiles == null) continue;
1654       for (StoreFile file : storeFiles) {
1655         StoreFile.Reader sfReader = file.getReader();
1656         if (sfReader == null) continue;
1657         HFile.Reader reader = sfReader.getHFileReader();
1658         if (reader == null) continue;
1659         if (majorCompactioOnly) {
1660           byte[] val = reader.loadFileInfo().get(StoreFile.MAJOR_COMPACTION_KEY);
1661           if (val == null) continue;
1662           if (val == null || !Bytes.toBoolean(val)) {
1663             continue;
1664           }
1665         }
1666         result = Math.min(result, reader.getFileContext().getFileCreateTime());
1667       }
1668     }
1669     return result == Long.MAX_VALUE ? 0 : result;
1670   }
1671 
1672   RegionLoad.Builder setCompleteSequenceId(RegionLoad.Builder regionLoadBldr) {
1673     long lastFlushOpSeqIdLocal = this.lastFlushOpSeqId;
1674     byte[] encodedRegionName = this.getRegionInfo().getEncodedNameAsBytes();
1675     regionLoadBldr.clearStoreCompleteSequenceId();
1676     for (byte[] familyName : this.stores.keySet()) {
1677       long earliest = this.wal.getEarliestMemstoreSeqNum(encodedRegionName, familyName);
1678       // Subtract - 1 to go earlier than the current oldest, unflushed edit in memstore; this will
1679       // give us a sequence id that is for sure flushed. We want edit replay to start after this
1680       // sequence id in this region. If NO_SEQNUM, use the regions maximum flush id.
1681       long csid = (earliest == HConstants.NO_SEQNUM)? lastFlushOpSeqIdLocal: earliest - 1;
1682       regionLoadBldr.addStoreCompleteSequenceId(StoreSequenceId.
1683         newBuilder().setFamilyName(ByteString.copyFrom(familyName)).setSequenceId(csid).build());
1684     }
1685     return regionLoadBldr.setCompleteSequenceId(getMaxFlushedSeqId());
1686   }
1687 
1688   //////////////////////////////////////////////////////////////////////////////
1689   // HRegion maintenance.
1690   //
1691   // These methods are meant to be called periodically by the HRegionServer for
1692   // upkeep.
1693   //////////////////////////////////////////////////////////////////////////////
1694 
1695   /** @return returns size of largest HStore. */
1696   public long getLargestHStoreSize() {
1697     long size = 0;
1698     for (Store h : stores.values()) {
1699       long storeSize = h.getSize();
1700       if (storeSize > size) {
1701         size = storeSize;
1702       }
1703     }
1704     return size;
1705   }
1706 
1707   /**
1708    * @return KeyValue Comparator
1709    */
1710   public KeyValue.KVComparator getComparator() {
1711     return this.comparator;
1712   }
1713 
1714   /*
1715    * Do preparation for pending compaction.
1716    * @throws IOException
1717    */
1718   protected void doRegionCompactionPrep() throws IOException {
1719   }
1720 
1721   @Override
1722   public void triggerMajorCompaction() throws IOException {
1723     for (Store s : getStores()) {
1724       s.triggerMajorCompaction();
1725     }
1726   }
1727 
1728   @Override
1729   public void compact(final boolean majorCompaction) throws IOException {
1730     if (majorCompaction) {
1731       triggerMajorCompaction();
1732     }
1733     for (Store s : getStores()) {
1734       CompactionContext compaction = s.requestCompaction();
1735       if (compaction != null) {
1736         CompactionThroughputController controller = null;
1737         if (rsServices != null) {
1738           controller = CompactionThroughputControllerFactory.create(rsServices, conf);
1739         }
1740         if (controller == null) {
1741           controller = NoLimitCompactionThroughputController.INSTANCE;
1742         }
1743         compact(compaction, s, controller, null);
1744       }
1745     }
1746   }
1747 
1748   /**
1749    * This is a helper function that compact all the stores synchronously
1750    * It is used by utilities and testing
1751    *
1752    * @throws IOException e
1753    */
1754   public void compactStores() throws IOException {
1755     for (Store s : getStores()) {
1756       CompactionContext compaction = s.requestCompaction();
1757       if (compaction != null) {
1758         compact(compaction, s, NoLimitCompactionThroughputController.INSTANCE, null);
1759       }
1760     }
1761   }
1762 
1763   /**
1764    * This is a helper function that compact the given store
1765    * It is used by utilities and testing
1766    *
1767    * @throws IOException e
1768    */
1769   @VisibleForTesting
1770   void compactStore(byte[] family, CompactionThroughputController throughputController)
1771       throws IOException {
1772     Store s = getStore(family);
1773     CompactionContext compaction = s.requestCompaction();
1774     if (compaction != null) {
1775       compact(compaction, s, throughputController, null);
1776     }
1777   }
1778 
1779   /*
1780    * Called by compaction thread and after region is opened to compact the
1781    * HStores if necessary.
1782    *
1783    * <p>This operation could block for a long time, so don't call it from a
1784    * time-sensitive thread.
1785    *
1786    * Note that no locking is necessary at this level because compaction only
1787    * conflicts with a region split, and that cannot happen because the region
1788    * server does them sequentially and not in parallel.
1789    *
1790    * @param compaction Compaction details, obtained by requestCompaction()
1791    * @param throughputController
1792    * @return whether the compaction completed
1793    */
1794   public boolean compact(CompactionContext compaction, Store store,
1795       CompactionThroughputController throughputController) throws IOException {
1796     return compact(compaction, store, throughputController, null);
1797   }
1798 
1799   public boolean compact(CompactionContext compaction, Store store,
1800       CompactionThroughputController throughputController, User user) throws IOException {
1801     assert compaction != null && compaction.hasSelection();
1802     assert !compaction.getRequest().getFiles().isEmpty();
1803     if (this.closing.get() || this.closed.get()) {
1804       LOG.debug("Skipping compaction on " + this + " because closing/closed");
1805       store.cancelRequestedCompaction(compaction);
1806       return false;
1807     }
1808     MonitoredTask status = null;
1809     boolean requestNeedsCancellation = true;
1810     // block waiting for the lock for compaction
1811     lock.readLock().lock();
1812     try {
1813       byte[] cf = Bytes.toBytes(store.getColumnFamilyName());
1814       if (stores.get(cf) != store) {
1815         LOG.warn("Store " + store.getColumnFamilyName() + " on region " + this
1816             + " has been re-instantiated, cancel this compaction request. "
1817             + " It may be caused by the roll back of split transaction");
1818         return false;
1819       }
1820 
1821       status = TaskMonitor.get().createStatus("Compacting " + store + " in " + this);
1822       if (this.closed.get()) {
1823         String msg = "Skipping compaction on " + this + " because closed";
1824         LOG.debug(msg);
1825         status.abort(msg);
1826         return false;
1827       }
1828       boolean wasStateSet = false;
1829       try {
1830         synchronized (writestate) {
1831           if (writestate.writesEnabled) {
1832             wasStateSet = true;
1833             writestate.compacting.incrementAndGet();
1834           } else {
1835             String msg = "NOT compacting region " + this + ". Writes disabled.";
1836             LOG.info(msg);
1837             status.abort(msg);
1838             return false;
1839           }
1840         }
1841         LOG.info("Starting compaction on " + store + " in region " + this
1842             + (compaction.getRequest().isOffPeak()?" as an off-peak compaction":""));
1843         doRegionCompactionPrep();
1844         try {
1845           status.setStatus("Compacting store " + store);
1846           // We no longer need to cancel the request on the way out of this
1847           // method because Store#compact will clean up unconditionally
1848           requestNeedsCancellation = false;
1849           store.compact(compaction, throughputController, user);
1850         } catch (InterruptedIOException iioe) {
1851           String msg = "compaction interrupted";
1852           LOG.info(msg, iioe);
1853           status.abort(msg);
1854           return false;
1855         }
1856       } finally {
1857         if (wasStateSet) {
1858           synchronized (writestate) {
1859             writestate.compacting.decrementAndGet();
1860             if (writestate.compacting.get() <= 0) {
1861               writestate.notifyAll();
1862             }
1863           }
1864         }
1865       }
1866       status.markComplete("Compaction complete");
1867       return true;
1868     } finally {
1869       try {
1870         if (requestNeedsCancellation) store.cancelRequestedCompaction(compaction);
1871         if (status != null) status.cleanup();
1872       } finally {
1873         lock.readLock().unlock();
1874       }
1875     }
1876   }
1877 
1878   @Override
1879   public FlushResult flush(boolean force) throws IOException {
1880     return flushcache(force, false);
1881   }
1882 
1883   /**
1884    * Flush the cache.
1885    *
1886    * When this method is called the cache will be flushed unless:
1887    * <ol>
1888    *   <li>the cache is empty</li>
1889    *   <li>the region is closed.</li>
1890    *   <li>a flush is already in progress</li>
1891    *   <li>writes are disabled</li>
1892    * </ol>
1893    *
1894    * <p>This method may block for some time, so it should not be called from a
1895    * time-sensitive thread.
1896    * @param forceFlushAllStores whether we want to flush all stores
1897    * @param writeFlushRequestWalMarker whether to write the flush request marker to WAL
1898    * @return whether the flush is success and whether the region needs compacting
1899    *
1900    * @throws IOException general io exceptions
1901    * @throws DroppedSnapshotException Thrown when replay of wal is required
1902    * because a Snapshot was not properly persisted. The region is put in closing mode, and the
1903    * caller MUST abort after this.
1904    */
1905   public FlushResult flushcache(boolean forceFlushAllStores, boolean writeFlushRequestWalMarker)
1906       throws IOException {
1907     // fail-fast instead of waiting on the lock
1908     if (this.closing.get()) {
1909       String msg = "Skipping flush on " + this + " because closing";
1910       LOG.debug(msg);
1911       return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false);
1912     }
1913     MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this);
1914     status.setStatus("Acquiring readlock on region");
1915     // block waiting for the lock for flushing cache
1916     lock.readLock().lock();
1917     try {
1918       if (this.closed.get()) {
1919         String msg = "Skipping flush on " + this + " because closed";
1920         LOG.debug(msg);
1921         status.abort(msg);
1922         return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false);
1923       }
1924       if (coprocessorHost != null) {
1925         status.setStatus("Running coprocessor pre-flush hooks");
1926         coprocessorHost.preFlush();
1927       }
1928       // TODO: this should be managed within memstore with the snapshot, updated only after flush
1929       // successful
1930       if (numMutationsWithoutWAL.get() > 0) {
1931         numMutationsWithoutWAL.set(0);
1932         dataInMemoryWithoutWAL.set(0);
1933       }
1934       synchronized (writestate) {
1935         if (!writestate.flushing && writestate.writesEnabled) {
1936           this.writestate.flushing = true;
1937         } else {
1938           if (LOG.isDebugEnabled()) {
1939             LOG.debug("NOT flushing memstore for region " + this
1940                 + ", flushing=" + writestate.flushing + ", writesEnabled="
1941                 + writestate.writesEnabled);
1942           }
1943           String msg = "Not flushing since "
1944               + (writestate.flushing ? "already flushing"
1945               : "writes not enabled");
1946           status.abort(msg);
1947           return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false);
1948         }
1949       }
1950 
1951       try {
1952         Collection<Store> specificStoresToFlush =
1953             forceFlushAllStores ? stores.values() : flushPolicy.selectStoresToFlush();
1954         FlushResult fs = internalFlushcache(specificStoresToFlush,
1955           status, writeFlushRequestWalMarker);
1956 
1957         if (coprocessorHost != null) {
1958           status.setStatus("Running post-flush coprocessor hooks");
1959           coprocessorHost.postFlush();
1960         }
1961 
1962         status.markComplete("Flush successful");
1963         return fs;
1964       } finally {
1965         synchronized (writestate) {
1966           writestate.flushing = false;
1967           this.writestate.flushRequested = false;
1968           writestate.notifyAll();
1969         }
1970       }
1971     } finally {
1972       lock.readLock().unlock();
1973       status.cleanup();
1974     }
1975   }
1976 
1977   /**
1978    * Should the store be flushed because it is old enough.
1979    * <p>
1980    * Every FlushPolicy should call this to determine whether a store is old enough to flush(except
1981    * that you always flush all stores). Otherwise the {@link #shouldFlush()} method will always
1982    * returns true which will make a lot of flush requests.
1983    */
1984   boolean shouldFlushStore(Store store) {
1985     long earliest = this.wal.getEarliestMemstoreSeqNum(getRegionInfo().getEncodedNameAsBytes(),
1986       store.getFamily().getName()) - 1;
1987     if (earliest > 0 && earliest + flushPerChanges < mvcc.getReadPoint()) {
1988       if (LOG.isDebugEnabled()) {
1989         LOG.debug("Flush column family " + store.getColumnFamilyName() + " of " +
1990           getRegionInfo().getEncodedName() + " because unflushed sequenceid=" + earliest +
1991           " is > " + this.flushPerChanges + " from current=" + mvcc.getReadPoint());
1992       }
1993       return true;
1994     }
1995     if (this.flushCheckInterval <= 0) {
1996       return false;
1997     }
1998     long now = EnvironmentEdgeManager.currentTime();
1999     if (store.timeOfOldestEdit() < now - this.flushCheckInterval) {
2000       if (LOG.isDebugEnabled()) {
2001         LOG.debug("Flush column family: " + store.getColumnFamilyName() + " of " +
2002           getRegionInfo().getEncodedName() + " because time of oldest edit=" +
2003             store.timeOfOldestEdit() + " is > " + this.flushCheckInterval + " from now =" + now);
2004       }
2005       return true;
2006     }
2007     return false;
2008   }
2009 
2010   /**
2011    * Should the memstore be flushed now
2012    */
2013   boolean shouldFlush(final StringBuffer whyFlush) {
2014     whyFlush.setLength(0);
2015     // This is a rough measure.
2016     if (this.maxFlushedSeqId > 0
2017           && (this.maxFlushedSeqId + this.flushPerChanges < this.mvcc.getReadPoint())) {
2018       whyFlush.append("more than max edits, " + this.flushPerChanges + ", since last flush");
2019       return true;
2020     }
2021     long modifiedFlushCheckInterval = flushCheckInterval;
2022     if (getRegionInfo().isSystemTable() &&
2023         getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2024       modifiedFlushCheckInterval = SYSTEM_CACHE_FLUSH_INTERVAL;
2025     }
2026     if (modifiedFlushCheckInterval <= 0) { //disabled
2027       return false;
2028     }
2029     long now = EnvironmentEdgeManager.currentTime();
2030     //if we flushed in the recent past, we don't need to do again now
2031     if ((now - getEarliestFlushTimeForAllStores() < modifiedFlushCheckInterval)) {
2032       return false;
2033     }
2034     //since we didn't flush in the recent past, flush now if certain conditions
2035     //are met. Return true on first such memstore hit.
2036     for (Store s : getStores()) {
2037       if (s.timeOfOldestEdit() < now - modifiedFlushCheckInterval) {
2038         // we have an old enough edit in the memstore, flush
2039         whyFlush.append(s.toString() + " has an old edit so flush to free WALs");
2040         return true;
2041       }
2042     }
2043     return false;
2044   }
2045 
2046   /**
2047    * Flushing all stores.
2048    *
2049    * @see #internalFlushcache(Collection, MonitoredTask, boolean)
2050    */
2051   private FlushResult internalFlushcache(MonitoredTask status)
2052       throws IOException {
2053     return internalFlushcache(stores.values(), status, false);
2054   }
2055 
2056   /**
2057    * Flushing given stores.
2058    *
2059    * @see #internalFlushcache(WAL, long, Collection, MonitoredTask, boolean)
2060    */
2061   private FlushResult internalFlushcache(final Collection<Store> storesToFlush,
2062       MonitoredTask status, boolean writeFlushWalMarker) throws IOException {
2063     return internalFlushcache(this.wal, HConstants.NO_SEQNUM, storesToFlush,
2064         status, writeFlushWalMarker);
2065   }
2066 
2067   /**
2068    * Flush the memstore. Flushing the memstore is a little tricky. We have a lot
2069    * of updates in the memstore, all of which have also been written to the wal.
2070    * We need to write those updates in the memstore out to disk, while being
2071    * able to process reads/writes as much as possible during the flush
2072    * operation.
2073    * <p>
2074    * This method may block for some time. Every time you call it, we up the
2075    * regions sequence id even if we don't flush; i.e. the returned region id
2076    * will be at least one larger than the last edit applied to this region. The
2077    * returned id does not refer to an actual edit. The returned id can be used
2078    * for say installing a bulk loaded file just ahead of the last hfile that was
2079    * the result of this flush, etc.
2080    *
2081    * @param wal
2082    *          Null if we're NOT to go via wal.
2083    * @param myseqid
2084    *          The seqid to use if <code>wal</code> is null writing out flush
2085    *          file.
2086    * @param storesToFlush
2087    *          The list of stores to flush.
2088    * @return object describing the flush's state
2089    * @throws IOException
2090    *           general io exceptions
2091    * @throws DroppedSnapshotException
2092    *           Thrown when replay of wal is required because a Snapshot was not
2093    *           properly persisted.
2094    */
2095   protected FlushResult internalFlushcache(final WAL wal, final long myseqid,
2096       final Collection<Store> storesToFlush, MonitoredTask status, boolean writeFlushWalMarker)
2097           throws IOException {
2098     PrepareFlushResult result
2099       = internalPrepareFlushCache(wal, myseqid, storesToFlush, status, writeFlushWalMarker);
2100     if (result.result == null) {
2101       return internalFlushCacheAndCommit(wal, status, result, storesToFlush);
2102     } else {
2103       return result.result; // early exit due to failure from prepare stage
2104     }
2105   }
2106 
2107   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="DLS_DEAD_LOCAL_STORE",
2108       justification="FindBugs seems confused about trxId")
2109   protected PrepareFlushResult internalPrepareFlushCache(final WAL wal, final long myseqid,
2110       final Collection<Store> storesToFlush, MonitoredTask status, boolean writeFlushWalMarker)
2111   throws IOException {
2112     if (this.rsServices != null && this.rsServices.isAborted()) {
2113       // Don't flush when server aborting, it's unsafe
2114       throw new IOException("Aborting flush because server is aborted...");
2115     }
2116     final long startTime = EnvironmentEdgeManager.currentTime();
2117     // If nothing to flush, return, but we need to safely update the region sequence id
2118     if (this.memstoreSize.get() <= 0) {
2119       // Take an update lock because am about to change the sequence id and we want the sequence id
2120       // to be at the border of the empty memstore.
2121       MultiVersionConcurrencyControl.WriteEntry writeEntry = null;
2122       this.updatesLock.writeLock().lock();
2123       try {
2124         if (this.memstoreSize.get() <= 0) {
2125           // Presume that if there are still no edits in the memstore, then there are no edits for
2126           // this region out in the WAL subsystem so no need to do any trickery clearing out
2127           // edits in the WAL system. Up the sequence number so the resulting flush id is for
2128           // sure just beyond the last appended region edit (useful as a marker when bulk loading,
2129           // etc.). NOTE: The writeEntry write number is NOT in the WAL.. there is no WAL writing
2130           // here.
2131           if (wal != null) {
2132             writeEntry = mvcc.begin();
2133             long flushOpSeqId = writeEntry.getWriteNumber();
2134             FlushResult flushResult = new FlushResultImpl(
2135                 FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY,
2136                 flushOpSeqId,
2137                 "Nothing to flush",
2138                 writeFlushRequestMarkerToWAL(wal, writeFlushWalMarker));
2139             // TODO: Lets see if we hang here, if there is a scenario where an outstanding reader
2140             // with a read point is in advance of this write point.
2141             mvcc.completeAndWait(writeEntry);
2142             writeEntry = null;
2143             return new PrepareFlushResult(flushResult, myseqid);
2144           } else {
2145             return new PrepareFlushResult(
2146               new FlushResultImpl(
2147                   FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY,
2148                   "Nothing to flush",
2149                   false),
2150               myseqid);
2151           }
2152         }
2153       } finally {
2154         this.updatesLock.writeLock().unlock();
2155         if (writeEntry != null) {
2156           mvcc.complete(writeEntry);
2157         }
2158       }
2159     }
2160 
2161     if (LOG.isInfoEnabled()) {
2162       // Log a fat line detailing what is being flushed.
2163       StringBuilder perCfExtras = null;
2164       if (!isAllFamilies(storesToFlush)) {
2165         perCfExtras = new StringBuilder();
2166         for (Store store: storesToFlush) {
2167           perCfExtras.append("; ").append(store.getColumnFamilyName());
2168           perCfExtras.append("=").append(StringUtils.byteDesc(store.getMemStoreSize()));
2169         }
2170       }
2171       LOG.info("Flushing " + + storesToFlush.size() + "/" + stores.size() +
2172         " column families, memstore=" + StringUtils.byteDesc(this.memstoreSize.get()) +
2173         ((perCfExtras != null && perCfExtras.length() > 0)? perCfExtras.toString(): "") +
2174         ((wal != null) ? "" : "; WAL is null, using passed sequenceid=" + myseqid));
2175     }
2176     // Stop updates while we snapshot the memstore of all of these regions' stores. We only have
2177     // to do this for a moment.  It is quick. We also set the memstore size to zero here before we
2178     // allow updates again so its value will represent the size of the updates received
2179     // during flush
2180 
2181     // We have to take an update lock during snapshot, or else a write could end up in both snapshot
2182     // and memstore (makes it difficult to do atomic rows then)
2183     status.setStatus("Obtaining lock to block concurrent updates");
2184     // block waiting for the lock for internal flush
2185     this.updatesLock.writeLock().lock();
2186     status.setStatus("Preparing to flush by snapshotting stores in " +
2187       getRegionInfo().getEncodedName());
2188     long totalFlushableSizeOfFlushableStores = 0;
2189 
2190     Set<byte[]> flushedFamilyNames = new HashSet<byte[]>();
2191     for (Store store: storesToFlush) {
2192       flushedFamilyNames.add(store.getFamily().getName());
2193     }
2194 
2195     TreeMap<byte[], StoreFlushContext> storeFlushCtxs
2196       = new TreeMap<byte[], StoreFlushContext>(Bytes.BYTES_COMPARATOR);
2197     TreeMap<byte[], List<Path>> committedFiles = new TreeMap<byte[], List<Path>>(
2198         Bytes.BYTES_COMPARATOR);
2199     TreeMap<byte[], Long> storeFlushableSize
2200         = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
2201     // The sequence id of this flush operation which is used to log FlushMarker and pass to
2202     // createFlushContext to use as the store file's sequence id. It can be in advance of edits
2203     // still in the memstore, edits that are in other column families yet to be flushed.
2204     long flushOpSeqId = HConstants.NO_SEQNUM;
2205     // The max flushed sequence id after this flush operation completes. All edits in memstore
2206     // will be in advance of this sequence id.
2207     long flushedSeqId = HConstants.NO_SEQNUM;
2208     byte[] encodedRegionName = getRegionInfo().getEncodedNameAsBytes();
2209 
2210     long trxId = 0;
2211     MultiVersionConcurrencyControl.WriteEntry writeEntry = mvcc.begin();
2212     try {
2213       try {
2214         if (wal != null) {
2215           Long earliestUnflushedSequenceIdForTheRegion =
2216             wal.startCacheFlush(encodedRegionName, flushedFamilyNames);
2217           if (earliestUnflushedSequenceIdForTheRegion == null) {
2218             // This should never happen. This is how startCacheFlush signals flush cannot proceed.
2219             String msg = this.getRegionInfo().getEncodedName() + " flush aborted; WAL closing.";
2220             status.setStatus(msg);
2221             return new PrepareFlushResult(
2222               new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false),
2223               myseqid);
2224           }
2225           flushOpSeqId = getNextSequenceId(wal);
2226           // Back up 1, minus 1 from oldest sequence id in memstore to get last 'flushed' edit
2227           flushedSeqId =
2228             earliestUnflushedSequenceIdForTheRegion.longValue() == HConstants.NO_SEQNUM?
2229               flushOpSeqId: earliestUnflushedSequenceIdForTheRegion.longValue() - 1;
2230         } else {
2231           // use the provided sequence Id as WAL is not being used for this flush.
2232           flushedSeqId = flushOpSeqId = myseqid;
2233         }
2234 
2235         for (Store s : storesToFlush) {
2236           totalFlushableSizeOfFlushableStores += s.getFlushableSize();
2237           storeFlushCtxs.put(s.getFamily().getName(), s.createFlushContext(flushOpSeqId));
2238           committedFiles.put(s.getFamily().getName(), null); // for writing stores to WAL
2239           storeFlushableSize.put(s.getFamily().getName(), s.getFlushableSize());
2240         }
2241 
2242         // write the snapshot start to WAL
2243         if (wal != null && !writestate.readOnly) {
2244           FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.START_FLUSH,
2245             getRegionInfo(), flushOpSeqId, committedFiles);
2246           // no sync. Sync is below where we do not hold the updates lock
2247           trxId = WALUtil.writeFlushMarker(wal, this.htableDescriptor, getRegionInfo(),
2248             desc, false, mvcc);
2249         }
2250 
2251         // Prepare flush (take a snapshot)
2252         for (StoreFlushContext flush : storeFlushCtxs.values()) {
2253           flush.prepare();
2254         }
2255       } catch (IOException ex) {
2256         if (wal != null) {
2257           if (trxId > 0) { // check whether we have already written START_FLUSH to WAL
2258             try {
2259               FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.ABORT_FLUSH,
2260                 getRegionInfo(), flushOpSeqId, committedFiles);
2261               WALUtil.writeFlushMarker(wal, this.htableDescriptor, getRegionInfo(),
2262                 desc, false, mvcc);
2263             } catch (Throwable t) {
2264               LOG.warn("Received unexpected exception trying to write ABORT_FLUSH marker to WAL:" +
2265                   StringUtils.stringifyException(t));
2266               // ignore this since we will be aborting the RS with DSE.
2267             }
2268           }
2269           // we have called wal.startCacheFlush(), now we have to abort it
2270           wal.abortCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());
2271           throw ex; // let upper layers deal with it.
2272         }
2273       } finally {
2274         this.updatesLock.writeLock().unlock();
2275       }
2276       String s = "Finished memstore snapshotting " + this +
2277         ", syncing WAL and waiting on mvcc, flushsize=" + totalFlushableSizeOfFlushableStores;
2278       status.setStatus(s);
2279       if (LOG.isTraceEnabled()) LOG.trace(s);
2280       // sync unflushed WAL changes
2281       // see HBASE-8208 for details
2282       if (wal != null) {
2283         try {
2284           wal.sync(); // ensure that flush marker is sync'ed
2285         } catch (IOException ioe) {
2286           wal.abortCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());
2287           throw ioe;
2288         }
2289       }
2290 
2291       // wait for all in-progress transactions to commit to WAL before
2292       // we can start the flush. This prevents
2293       // uncommitted transactions from being written into HFiles.
2294       // We have to block before we start the flush, otherwise keys that
2295       // were removed via a rollbackMemstore could be written to Hfiles.
2296       mvcc.completeAndWait(writeEntry);
2297       // set writeEntry to null to prevent mvcc.complete from being called again inside finally
2298       // block
2299       writeEntry = null;
2300     } finally {
2301       if (writeEntry != null) {
2302         // In case of failure just mark current writeEntry as complete.
2303         mvcc.complete(writeEntry);
2304       }
2305     }
2306     return new PrepareFlushResult(storeFlushCtxs, committedFiles, storeFlushableSize, startTime,
2307         flushOpSeqId, flushedSeqId, totalFlushableSizeOfFlushableStores);
2308   }
2309 
2310   /**
2311    * @param families
2312    * @return True if passed Set is all families in the region.
2313    */
2314   private boolean isAllFamilies(final Collection<Store> families) {
2315     return families == null || this.stores.size() == families.size();
2316   }
2317 
2318   /**
2319    * Writes a marker to WAL indicating a flush is requested but cannot be complete due to various
2320    * reasons. Ignores exceptions from WAL. Returns whether the write succeeded.
2321    * @param wal
2322    * @return whether WAL write was successful
2323    */
2324   private boolean writeFlushRequestMarkerToWAL(WAL wal, boolean writeFlushWalMarker) {
2325     if (writeFlushWalMarker && wal != null && !writestate.readOnly) {
2326       FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.CANNOT_FLUSH,
2327         getRegionInfo(), -1, new TreeMap<byte[], List<Path>>(Bytes.BYTES_COMPARATOR));
2328       try {
2329         WALUtil.writeFlushMarker(wal, this.htableDescriptor, getRegionInfo(),
2330           desc, true, mvcc);
2331         return true;
2332       } catch (IOException e) {
2333         LOG.warn(getRegionInfo().getEncodedName() + " : "
2334             + "Received exception while trying to write the flush request to wal", e);
2335       }
2336     }
2337     return false;
2338   }
2339 
2340   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NN_NAKED_NOTIFY",
2341       justification="Intentional; notify is about completed flush")
2342   protected FlushResult internalFlushCacheAndCommit(
2343         final WAL wal, MonitoredTask status, final PrepareFlushResult prepareResult,
2344         final Collection<Store> storesToFlush)
2345     throws IOException {
2346 
2347     // prepare flush context is carried via PrepareFlushResult
2348     TreeMap<byte[], StoreFlushContext> storeFlushCtxs = prepareResult.storeFlushCtxs;
2349     TreeMap<byte[], List<Path>> committedFiles = prepareResult.committedFiles;
2350     long startTime = prepareResult.startTime;
2351     long flushOpSeqId = prepareResult.flushOpSeqId;
2352     long flushedSeqId = prepareResult.flushedSeqId;
2353     long totalFlushableSizeOfFlushableStores = prepareResult.totalFlushableSize;
2354 
2355     String s = "Flushing stores of " + this;
2356     status.setStatus(s);
2357     if (LOG.isTraceEnabled()) LOG.trace(s);
2358 
2359     // Any failure from here on out will be catastrophic requiring server
2360     // restart so wal content can be replayed and put back into the memstore.
2361     // Otherwise, the snapshot content while backed up in the wal, it will not
2362     // be part of the current running servers state.
2363     boolean compactionRequested = false;
2364     try {
2365       // A.  Flush memstore to all the HStores.
2366       // Keep running vector of all store files that includes both old and the
2367       // just-made new flush store file. The new flushed file is still in the
2368       // tmp directory.
2369 
2370       for (StoreFlushContext flush : storeFlushCtxs.values()) {
2371         flush.flushCache(status);
2372       }
2373 
2374       // Switch snapshot (in memstore) -> new hfile (thus causing
2375       // all the store scanners to reset/reseek).
2376       Iterator<Store> it = storesToFlush.iterator();
2377       // stores.values() and storeFlushCtxs have same order
2378       for (StoreFlushContext flush : storeFlushCtxs.values()) {
2379         boolean needsCompaction = flush.commit(status);
2380         if (needsCompaction) {
2381           compactionRequested = true;
2382         }
2383         byte[] storeName = it.next().getFamily().getName();
2384         List<Path> storeCommittedFiles = flush.getCommittedFiles();
2385         committedFiles.put(storeName, storeCommittedFiles);
2386         // Flush committed no files, indicating flush is empty or flush was canceled
2387         if (storeCommittedFiles == null || storeCommittedFiles.isEmpty()) {
2388           totalFlushableSizeOfFlushableStores -= prepareResult.storeFlushableSize.get(storeName);
2389         }
2390       }
2391       storeFlushCtxs.clear();
2392 
2393       // Set down the memstore size by amount of flush.
2394       this.addAndGetGlobalMemstoreSize(-totalFlushableSizeOfFlushableStores);
2395 
2396       if (wal != null) {
2397         // write flush marker to WAL. If fail, we should throw DroppedSnapshotException
2398         FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.COMMIT_FLUSH,
2399           getRegionInfo(), flushOpSeqId, committedFiles);
2400         WALUtil.writeFlushMarker(wal, this.htableDescriptor, getRegionInfo(),
2401           desc, true, mvcc);
2402       }
2403     } catch (Throwable t) {
2404       // An exception here means that the snapshot was not persisted.
2405       // The wal needs to be replayed so its content is restored to memstore.
2406       // Currently, only a server restart will do this.
2407       // We used to only catch IOEs but its possible that we'd get other
2408       // exceptions -- e.g. HBASE-659 was about an NPE -- so now we catch
2409       // all and sundry.
2410       if (wal != null) {
2411         try {
2412           FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.ABORT_FLUSH,
2413             getRegionInfo(), flushOpSeqId, committedFiles);
2414           WALUtil.writeFlushMarker(wal, this.htableDescriptor, getRegionInfo(),
2415             desc, false, mvcc);
2416         } catch (Throwable ex) {
2417           LOG.warn(getRegionInfo().getEncodedName() + " : "
2418               + "failed writing ABORT_FLUSH marker to WAL", ex);
2419           // ignore this since we will be aborting the RS with DSE.
2420         }
2421         wal.abortCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());
2422       }
2423       DroppedSnapshotException dse = new DroppedSnapshotException("region: " +
2424           Bytes.toStringBinary(getRegionInfo().getRegionName()));
2425       dse.initCause(t);
2426       status.abort("Flush failed: " + StringUtils.stringifyException(t));
2427 
2428       // Callers for flushcache() should catch DroppedSnapshotException and abort the region server.
2429       // However, since we may have the region read lock, we cannot call close(true) here since
2430       // we cannot promote to a write lock. Instead we are setting closing so that all other region
2431       // operations except for close will be rejected.
2432       this.closing.set(true);
2433 
2434       if (rsServices != null) {
2435         // This is a safeguard against the case where the caller fails to explicitly handle aborting
2436         rsServices.abort("Replay of WAL required. Forcing server shutdown", dse);
2437       }
2438 
2439       throw dse;
2440     }
2441 
2442     // If we get to here, the HStores have been written.
2443     if (wal != null) {
2444       wal.completeCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());
2445     }
2446 
2447     // Record latest flush time
2448     for (Store store: storesToFlush) {
2449       this.lastStoreFlushTimeMap.put(store, startTime);
2450     }
2451 
2452     this.maxFlushedSeqId = flushedSeqId;
2453     this.lastFlushOpSeqId = flushOpSeqId;
2454 
2455     // C. Finally notify anyone waiting on memstore to clear:
2456     // e.g. checkResources().
2457     synchronized (this) {
2458       notifyAll(); // FindBugs NN_NAKED_NOTIFY
2459     }
2460 
2461     long time = EnvironmentEdgeManager.currentTime() - startTime;
2462     long memstoresize = this.memstoreSize.get();
2463     String msg = "Finished memstore flush of ~"
2464         + StringUtils.byteDesc(totalFlushableSizeOfFlushableStores) + "/"
2465         + totalFlushableSizeOfFlushableStores + ", currentsize="
2466         + StringUtils.byteDesc(memstoresize) + "/" + memstoresize
2467         + " for region " + this + " in " + time + "ms, sequenceid="
2468         + flushOpSeqId +  ", compaction requested=" + compactionRequested
2469         + ((wal == null) ? "; wal=null" : "");
2470     LOG.info(msg);
2471     status.setStatus(msg);
2472 
2473     return new FlushResultImpl(compactionRequested ?
2474         FlushResult.Result.FLUSHED_COMPACTION_NEEDED :
2475           FlushResult.Result.FLUSHED_NO_COMPACTION_NEEDED,
2476         flushOpSeqId);
2477   }
2478 
2479   /**
2480    * Method to safely get the next sequence number.
2481    * @return Next sequence number unassociated with any actual edit.
2482    * @throws IOException
2483    */
2484   @VisibleForTesting
2485   protected long getNextSequenceId(final WAL wal) throws IOException {
2486     // TODO: For review. Putting an empty edit in to get a sequenceid out will not work if the
2487     // WAL is banjaxed... if it has gotten an exception and the WAL has not yet been rolled or
2488     // aborted. In this case, we'll just get stuck here. For now, until HBASE-12751, just have
2489     // a timeout. May happen in tests after we tightened the semantic via HBASE-14317.
2490     // Also, the getSequenceId blocks on a latch. There is no global list of outstanding latches
2491     // so if an abort or stop, there is no way to call them in.
2492     WALKey key = this.appendEmptyEdit(wal);
2493     mvcc.complete(key.getWriteEntry());
2494     return key.getSequenceId(this.maxWaitForSeqId);
2495   }
2496 
2497   //////////////////////////////////////////////////////////////////////////////
2498   // get() methods for client use.
2499   //////////////////////////////////////////////////////////////////////////////
2500 
2501   @Override
2502   public Result getClosestRowBefore(final byte [] row, final byte [] family) throws IOException {
2503     if (coprocessorHost != null) {
2504       Result result = new Result();
2505       if (coprocessorHost.preGetClosestRowBefore(row, family, result)) {
2506         return result;
2507       }
2508     }
2509     // look across all the HStores for this region and determine what the
2510     // closest key is across all column families, since the data may be sparse
2511     checkRow(row, "getClosestRowBefore");
2512     startRegionOperation(Operation.GET);
2513     this.readRequestsCount.increment();
2514     try {
2515       Store store = getStore(family);
2516       // get the closest key. (HStore.getRowKeyAtOrBefore can return null)
2517       Cell key = store.getRowKeyAtOrBefore(row);
2518       Result result = null;
2519       if (key != null) {
2520         Get get = new Get(CellUtil.cloneRow(key));
2521         get.addFamily(family);
2522         result = get(get);
2523       }
2524       if (coprocessorHost != null) {
2525         coprocessorHost.postGetClosestRowBefore(row, family, result);
2526       }
2527       return result;
2528     } finally {
2529       closeRegionOperation(Operation.GET);
2530     }
2531   }
2532 
2533   @Override
2534   public RegionScanner getScanner(Scan scan) throws IOException {
2535    return getScanner(scan, null);
2536   }
2537 
2538   @Override
2539   public RegionScanner getScanner(Scan scan,
2540       List<KeyValueScanner> additionalScanners) throws IOException {
2541     startRegionOperation(Operation.SCAN);
2542     try {
2543       // Verify families are all valid
2544       if (!scan.hasFamilies()) {
2545         // Adding all families to scanner
2546         for (byte[] family: this.htableDescriptor.getFamiliesKeys()) {
2547           scan.addFamily(family);
2548         }
2549       } else {
2550         for (byte [] family : scan.getFamilyMap().keySet()) {
2551           checkFamily(family);
2552         }
2553       }
2554       return instantiateRegionScanner(scan, additionalScanners);
2555     } finally {
2556       closeRegionOperation(Operation.SCAN);
2557     }
2558   }
2559 
2560   protected RegionScanner instantiateRegionScanner(Scan scan,
2561       List<KeyValueScanner> additionalScanners) throws IOException {
2562     if (scan.isReversed()) {
2563       if (scan.getFilter() != null) {
2564         scan.getFilter().setReversed(true);
2565       }
2566       return new ReversedRegionScannerImpl(scan, additionalScanners, this);
2567     }
2568     return new RegionScannerImpl(scan, additionalScanners, this);
2569   }
2570 
2571   @Override
2572   public void prepareDelete(Delete delete) throws IOException {
2573     // Check to see if this is a deleteRow insert
2574     if(delete.getFamilyCellMap().isEmpty()){
2575       for(byte [] family : this.htableDescriptor.getFamiliesKeys()){
2576         // Don't eat the timestamp
2577         delete.addFamily(family, delete.getTimeStamp());
2578       }
2579     } else {
2580       for(byte [] family : delete.getFamilyCellMap().keySet()) {
2581         if(family == null) {
2582           throw new NoSuchColumnFamilyException("Empty family is invalid");
2583         }
2584         checkFamily(family);
2585       }
2586     }
2587   }
2588 
2589   @Override
2590   public void delete(Delete delete) throws IOException {
2591     checkReadOnly();
2592     checkResources();
2593     startRegionOperation(Operation.DELETE);
2594     try {
2595       delete.getRow();
2596       // All edits for the given row (across all column families) must happen atomically.
2597       doBatchMutate(delete);
2598     } finally {
2599       closeRegionOperation(Operation.DELETE);
2600     }
2601   }
2602 
2603   /**
2604    * Row needed by below method.
2605    */
2606   private static final byte [] FOR_UNIT_TESTS_ONLY = Bytes.toBytes("ForUnitTestsOnly");
2607 
2608   /**
2609    * This is used only by unit tests. Not required to be a public API.
2610    * @param familyMap map of family to edits for the given family.
2611    * @throws IOException
2612    */
2613   void delete(NavigableMap<byte[], List<Cell>> familyMap,
2614       Durability durability) throws IOException {
2615     Delete delete = new Delete(FOR_UNIT_TESTS_ONLY);
2616     delete.setFamilyCellMap(familyMap);
2617     delete.setDurability(durability);
2618     doBatchMutate(delete);
2619   }
2620 
2621   @Override
2622   public void prepareDeleteTimestamps(Mutation mutation, Map<byte[], List<Cell>> familyMap,
2623       byte[] byteNow) throws IOException {
2624     for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {
2625 
2626       byte[] family = e.getKey();
2627       List<Cell> cells = e.getValue();
2628       assert cells instanceof RandomAccess;
2629 
2630       Map<byte[], Integer> kvCount = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
2631       int listSize = cells.size();
2632       for (int i=0; i < listSize; i++) {
2633         Cell cell = cells.get(i);
2634         //  Check if time is LATEST, change to time of most recent addition if so
2635         //  This is expensive.
2636         if (cell.getTimestamp() == HConstants.LATEST_TIMESTAMP && CellUtil.isDeleteType(cell)) {
2637           byte[] qual = CellUtil.cloneQualifier(cell);
2638           if (qual == null) qual = HConstants.EMPTY_BYTE_ARRAY;
2639 
2640           Integer count = kvCount.get(qual);
2641           if (count == null) {
2642             kvCount.put(qual, 1);
2643           } else {
2644             kvCount.put(qual, count + 1);
2645           }
2646           count = kvCount.get(qual);
2647 
2648           Get get = new Get(CellUtil.cloneRow(cell));
2649           get.setMaxVersions(count);
2650           get.addColumn(family, qual);
2651           if (coprocessorHost != null) {
2652             if (!coprocessorHost.prePrepareTimeStampForDeleteVersion(mutation, cell,
2653                 byteNow, get)) {
2654               updateDeleteLatestVersionTimeStamp(cell, get, count, byteNow);
2655             }
2656           } else {
2657             updateDeleteLatestVersionTimeStamp(cell, get, count, byteNow);
2658           }
2659         } else {
2660           CellUtil.updateLatestStamp(cell, byteNow, 0);
2661         }
2662       }
2663     }
2664   }
2665 
2666   void updateDeleteLatestVersionTimeStamp(Cell cell, Get get, int count, byte[] byteNow)
2667       throws IOException {
2668     List<Cell> result = get(get, false);
2669 
2670     if (result.size() < count) {
2671       // Nothing to delete
2672       CellUtil.updateLatestStamp(cell, byteNow, 0);
2673       return;
2674     }
2675     if (result.size() > count) {
2676       throw new RuntimeException("Unexpected size: " + result.size());
2677     }
2678     Cell getCell = result.get(count - 1);
2679     CellUtil.setTimestamp(cell, getCell.getTimestamp());
2680   }
2681 
2682   @Override
2683   public void put(Put put) throws IOException {
2684     checkReadOnly();
2685 
2686     // Do a rough check that we have resources to accept a write.  The check is
2687     // 'rough' in that between the resource check and the call to obtain a
2688     // read lock, resources may run out.  For now, the thought is that this
2689     // will be extremely rare; we'll deal with it when it happens.
2690     checkResources();
2691     startRegionOperation(Operation.PUT);
2692     try {
2693       // All edits for the given row (across all column families) must happen atomically.
2694       doBatchMutate(put);
2695     } finally {
2696       closeRegionOperation(Operation.PUT);
2697     }
2698   }
2699 
2700   /**
2701    * Struct-like class that tracks the progress of a batch operation,
2702    * accumulating status codes and tracking the index at which processing
2703    * is proceeding.
2704    */
2705   private abstract static class BatchOperationInProgress<T> {
2706     T[] operations;
2707     int nextIndexToProcess = 0;
2708     OperationStatus[] retCodeDetails;
2709     WALEdit[] walEditsFromCoprocessors;
2710 
2711     public BatchOperationInProgress(T[] operations) {
2712       this.operations = operations;
2713       this.retCodeDetails = new OperationStatus[operations.length];
2714       this.walEditsFromCoprocessors = new WALEdit[operations.length];
2715       Arrays.fill(this.retCodeDetails, OperationStatus.NOT_RUN);
2716     }
2717 
2718     public abstract Mutation getMutation(int index);
2719     public abstract long getNonceGroup(int index);
2720     public abstract long getNonce(int index);
2721     /** This method is potentially expensive and should only be used for non-replay CP path. */
2722     public abstract Mutation[] getMutationsForCoprocs();
2723     public abstract boolean isInReplay();
2724     public abstract long getReplaySequenceId();
2725 
2726     public boolean isDone() {
2727       return nextIndexToProcess == operations.length;
2728     }
2729   }
2730 
2731   private static class MutationBatch extends BatchOperationInProgress<Mutation> {
2732     private long nonceGroup;
2733     private long nonce;
2734     public MutationBatch(Mutation[] operations, long nonceGroup, long nonce) {
2735       super(operations);
2736       this.nonceGroup = nonceGroup;
2737       this.nonce = nonce;
2738     }
2739 
2740     @Override
2741     public Mutation getMutation(int index) {
2742       return this.operations[index];
2743     }
2744 
2745     @Override
2746     public long getNonceGroup(int index) {
2747       return nonceGroup;
2748     }
2749 
2750     @Override
2751     public long getNonce(int index) {
2752       return nonce;
2753     }
2754 
2755     @Override
2756     public Mutation[] getMutationsForCoprocs() {
2757       return this.operations;
2758     }
2759 
2760     @Override
2761     public boolean isInReplay() {
2762       return false;
2763     }
2764 
2765     @Override
2766     public long getReplaySequenceId() {
2767       return 0;
2768     }
2769   }
2770 
2771   private static class ReplayBatch extends BatchOperationInProgress<MutationReplay> {
2772     private long replaySeqId = 0;
2773     public ReplayBatch(MutationReplay[] operations, long seqId) {
2774       super(operations);
2775       this.replaySeqId = seqId;
2776     }
2777 
2778     @Override
2779     public Mutation getMutation(int index) {
2780       return this.operations[index].mutation;
2781     }
2782 
2783     @Override
2784     public long getNonceGroup(int index) {
2785       return this.operations[index].nonceGroup;
2786     }
2787 
2788     @Override
2789     public long getNonce(int index) {
2790       return this.operations[index].nonce;
2791     }
2792 
2793     @Override
2794     public Mutation[] getMutationsForCoprocs() {
2795       assert false;
2796       throw new RuntimeException("Should not be called for replay batch");
2797     }
2798 
2799     @Override
2800     public boolean isInReplay() {
2801       return true;
2802     }
2803 
2804     @Override
2805     public long getReplaySequenceId() {
2806       return this.replaySeqId;
2807     }
2808   }
2809 
2810   @Override
2811   public OperationStatus[] batchMutate(Mutation[] mutations, long nonceGroup, long nonce)
2812       throws IOException {
2813     // As it stands, this is used for 3 things
2814     //  * batchMutate with single mutation - put/delete, separate or from checkAndMutate.
2815     //  * coprocessor calls (see ex. BulkDeleteEndpoint).
2816     // So nonces are not really ever used by HBase. They could be by coprocs, and checkAnd...
2817     return batchMutate(new MutationBatch(mutations, nonceGroup, nonce));
2818   }
2819 
2820   public OperationStatus[] batchMutate(Mutation[] mutations) throws IOException {
2821     return batchMutate(mutations, HConstants.NO_NONCE, HConstants.NO_NONCE);
2822   }
2823 
2824   @Override
2825   public OperationStatus[] batchReplay(MutationReplay[] mutations, long replaySeqId)
2826       throws IOException {
2827     if (!RegionReplicaUtil.isDefaultReplica(getRegionInfo())
2828         && replaySeqId < lastReplayedOpenRegionSeqId) {
2829       // if it is a secondary replica we should ignore these entries silently
2830       // since they are coming out of order
2831       if (LOG.isTraceEnabled()) {
2832         LOG.trace(getRegionInfo().getEncodedName() + " : "
2833           + "Skipping " + mutations.length + " mutations with replaySeqId=" + replaySeqId
2834           + " which is < than lastReplayedOpenRegionSeqId=" + lastReplayedOpenRegionSeqId);
2835         for (MutationReplay mut : mutations) {
2836           LOG.trace(getRegionInfo().getEncodedName() + " : Skipping : " + mut.mutation);
2837         }
2838       }
2839 
2840       OperationStatus[] statuses = new OperationStatus[mutations.length];
2841       for (int i = 0; i < statuses.length; i++) {
2842         statuses[i] = OperationStatus.SUCCESS;
2843       }
2844       return statuses;
2845     }
2846     return batchMutate(new ReplayBatch(mutations, replaySeqId));
2847   }
2848 
2849   /**
2850    * Perform a batch of mutations.
2851    * It supports only Put and Delete mutations and will ignore other types passed.
2852    * @param batchOp contains the list of mutations
2853    * @return an array of OperationStatus which internally contains the
2854    *         OperationStatusCode and the exceptionMessage if any.
2855    * @throws IOException
2856    */
2857   OperationStatus[] batchMutate(BatchOperationInProgress<?> batchOp) throws IOException {
2858     boolean initialized = false;
2859     Operation op = batchOp.isInReplay() ? Operation.REPLAY_BATCH_MUTATE : Operation.BATCH_MUTATE;
2860     startRegionOperation(op);
2861     try {
2862       while (!batchOp.isDone()) {
2863         if (!batchOp.isInReplay()) {
2864           checkReadOnly();
2865         }
2866         checkResources();
2867 
2868         if (!initialized) {
2869           this.writeRequestsCount.add(batchOp.operations.length);
2870           if (!batchOp.isInReplay()) {
2871             doPreMutationHook(batchOp);
2872           }
2873           initialized = true;
2874         }
2875         long addedSize = doMiniBatchMutation(batchOp);
2876         long newSize = this.addAndGetGlobalMemstoreSize(addedSize);
2877         if (isFlushSize(newSize)) {
2878           requestFlush();
2879         }
2880       }
2881     } finally {
2882       closeRegionOperation(op);
2883     }
2884     return batchOp.retCodeDetails;
2885   }
2886 
2887 
2888   private void doPreMutationHook(BatchOperationInProgress<?> batchOp)
2889       throws IOException {
2890     /* Run coprocessor pre hook outside of locks to avoid deadlock */
2891     WALEdit walEdit = new WALEdit();
2892     if (coprocessorHost != null) {
2893       for (int i = 0 ; i < batchOp.operations.length; i++) {
2894         Mutation m = batchOp.getMutation(i);
2895         if (m instanceof Put) {
2896           if (coprocessorHost.prePut((Put) m, walEdit, m.getDurability())) {
2897             // pre hook says skip this Put
2898             // mark as success and skip in doMiniBatchMutation
2899             batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
2900           }
2901         } else if (m instanceof Delete) {
2902           Delete curDel = (Delete) m;
2903           if (curDel.getFamilyCellMap().isEmpty()) {
2904             // handle deleting a row case
2905             prepareDelete(curDel);
2906           }
2907           if (coprocessorHost.preDelete(curDel, walEdit, m.getDurability())) {
2908             // pre hook says skip this Delete
2909             // mark as success and skip in doMiniBatchMutation
2910             batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
2911           }
2912         } else {
2913           // In case of passing Append mutations along with the Puts and Deletes in batchMutate
2914           // mark the operation return code as failure so that it will not be considered in
2915           // the doMiniBatchMutation
2916           batchOp.retCodeDetails[i] = new OperationStatus(OperationStatusCode.FAILURE,
2917               "Put/Delete mutations only supported in batchMutate() now");
2918         }
2919         if (!walEdit.isEmpty()) {
2920           batchOp.walEditsFromCoprocessors[i] = walEdit;
2921           walEdit = new WALEdit();
2922         }
2923       }
2924     }
2925   }
2926 
2927   @SuppressWarnings("unchecked")
2928   private long doMiniBatchMutation(BatchOperationInProgress<?> batchOp) throws IOException {
2929     boolean isInReplay = batchOp.isInReplay();
2930     // variable to note if all Put items are for the same CF -- metrics related
2931     boolean putsCfSetConsistent = true;
2932     //The set of columnFamilies first seen for Put.
2933     Set<byte[]> putsCfSet = null;
2934     // variable to note if all Delete items are for the same CF -- metrics related
2935     boolean deletesCfSetConsistent = true;
2936     //The set of columnFamilies first seen for Delete.
2937     Set<byte[]> deletesCfSet = null;
2938 
2939     long currentNonceGroup = HConstants.NO_NONCE, currentNonce = HConstants.NO_NONCE;
2940     WALEdit walEdit = new WALEdit(isInReplay);
2941     MultiVersionConcurrencyControl.WriteEntry writeEntry = null;
2942     long txid = 0;
2943     boolean doRollBackMemstore = false;
2944     boolean locked = false;
2945 
2946     /** Keep track of the locks we hold so we can release them in finally clause */
2947     List<RowLock> acquiredRowLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
2948     // reference family maps directly so coprocessors can mutate them if desired
2949     Map<byte[], List<Cell>>[] familyMaps = new Map[batchOp.operations.length];
2950     // We try to set up a batch in the range [firstIndex,lastIndexExclusive)
2951     int firstIndex = batchOp.nextIndexToProcess;
2952     int lastIndexExclusive = firstIndex;
2953     boolean success = false;
2954     int noOfPuts = 0, noOfDeletes = 0;
2955     WALKey walKey = null;
2956     long mvccNum = 0;
2957     try {
2958       // ------------------------------------
2959       // STEP 1. Try to acquire as many locks as we can, and ensure
2960       // we acquire at least one.
2961       // ----------------------------------
2962       int numReadyToWrite = 0;
2963       long now = EnvironmentEdgeManager.currentTime();
2964       while (lastIndexExclusive < batchOp.operations.length) {
2965         Mutation mutation = batchOp.getMutation(lastIndexExclusive);
2966         boolean isPutMutation = mutation instanceof Put;
2967 
2968         Map<byte[], List<Cell>> familyMap = mutation.getFamilyCellMap();
2969         // store the family map reference to allow for mutations
2970         familyMaps[lastIndexExclusive] = familyMap;
2971 
2972         // skip anything that "ran" already
2973         if (batchOp.retCodeDetails[lastIndexExclusive].getOperationStatusCode()
2974             != OperationStatusCode.NOT_RUN) {
2975           lastIndexExclusive++;
2976           continue;
2977         }
2978 
2979         try {
2980           if (isPutMutation) {
2981             // Check the families in the put. If bad, skip this one.
2982             if (isInReplay) {
2983               removeNonExistentColumnFamilyForReplay(familyMap);
2984             } else {
2985               checkFamilies(familyMap.keySet());
2986             }
2987             checkTimestamps(mutation.getFamilyCellMap(), now);
2988           } else {
2989             prepareDelete((Delete) mutation);
2990           }
2991           checkRow(mutation.getRow(), "doMiniBatchMutation");
2992         } catch (NoSuchColumnFamilyException nscf) {
2993           LOG.warn("No such column family in batch mutation", nscf);
2994           batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(
2995               OperationStatusCode.BAD_FAMILY, nscf.getMessage());
2996           lastIndexExclusive++;
2997           continue;
2998         } catch (FailedSanityCheckException fsce) {
2999           LOG.warn("Batch Mutation did not pass sanity check", fsce);
3000           batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(
3001               OperationStatusCode.SANITY_CHECK_FAILURE, fsce.getMessage());
3002           lastIndexExclusive++;
3003           continue;
3004         } catch (WrongRegionException we) {
3005           LOG.warn("Batch mutation had a row that does not belong to this region", we);
3006           batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(
3007               OperationStatusCode.SANITY_CHECK_FAILURE, we.getMessage());
3008           lastIndexExclusive++;
3009           continue;
3010         }
3011 
3012         // If we haven't got any rows in our batch, we should block to
3013         // get the next one.
3014         RowLock rowLock = null;
3015         try {
3016           rowLock = getRowLock(mutation.getRow(), true);
3017         } catch (IOException ioe) {
3018           LOG.warn("Failed getting lock in batch put, row="
3019             + Bytes.toStringBinary(mutation.getRow()), ioe);
3020         }
3021         if (rowLock == null) {
3022           // We failed to grab another lock
3023           break; // stop acquiring more rows for this batch
3024         } else {
3025           acquiredRowLocks.add(rowLock);
3026         }
3027 
3028         lastIndexExclusive++;
3029         numReadyToWrite++;
3030 
3031         if (isPutMutation) {
3032           // If Column Families stay consistent through out all of the
3033           // individual puts then metrics can be reported as a mutliput across
3034           // column families in the first put.
3035           if (putsCfSet == null) {
3036             putsCfSet = mutation.getFamilyCellMap().keySet();
3037           } else {
3038             putsCfSetConsistent = putsCfSetConsistent
3039                 && mutation.getFamilyCellMap().keySet().equals(putsCfSet);
3040           }
3041         } else {
3042           if (deletesCfSet == null) {
3043             deletesCfSet = mutation.getFamilyCellMap().keySet();
3044           } else {
3045             deletesCfSetConsistent = deletesCfSetConsistent
3046                 && mutation.getFamilyCellMap().keySet().equals(deletesCfSet);
3047           }
3048         }
3049       }
3050 
3051       // we should record the timestamp only after we have acquired the rowLock,
3052       // otherwise, newer puts/deletes are not guaranteed to have a newer timestamp
3053       now = EnvironmentEdgeManager.currentTime();
3054       byte[] byteNow = Bytes.toBytes(now);
3055 
3056       // Nothing to put/delete -- an exception in the above such as NoSuchColumnFamily?
3057       if (numReadyToWrite <= 0) return 0L;
3058 
3059       // We've now grabbed as many mutations off the list as we can
3060 
3061       // ------------------------------------
3062       // STEP 2. Update any LATEST_TIMESTAMP timestamps
3063       // ----------------------------------
3064       for (int i = firstIndex; !isInReplay && i < lastIndexExclusive; i++) {
3065         // skip invalid
3066         if (batchOp.retCodeDetails[i].getOperationStatusCode()
3067             != OperationStatusCode.NOT_RUN) continue;
3068 
3069         Mutation mutation = batchOp.getMutation(i);
3070         if (mutation instanceof Put) {
3071           updateCellTimestamps(familyMaps[i].values(), byteNow);
3072           noOfPuts++;
3073         } else {
3074           prepareDeleteTimestamps(mutation, familyMaps[i], byteNow);
3075           noOfDeletes++;
3076         }
3077         rewriteCellTags(familyMaps[i], mutation);
3078       }
3079 
3080       lock(this.updatesLock.readLock(), numReadyToWrite);
3081       locked = true;
3082 
3083       // calling the pre CP hook for batch mutation
3084       if (!isInReplay && coprocessorHost != null) {
3085         MiniBatchOperationInProgress<Mutation> miniBatchOp =
3086           new MiniBatchOperationInProgress<Mutation>(batchOp.getMutationsForCoprocs(),
3087           batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, lastIndexExclusive);
3088         if (coprocessorHost.preBatchMutate(miniBatchOp)) return 0L;
3089       }
3090 
3091       // ------------------------------------
3092       // STEP 3. Build WAL edit
3093       // ----------------------------------
3094       Durability durability = Durability.USE_DEFAULT;
3095       for (int i = firstIndex; i < lastIndexExclusive; i++) {
3096         // Skip puts that were determined to be invalid during preprocessing
3097         if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.NOT_RUN) {
3098           continue;
3099         }
3100 
3101         Mutation m = batchOp.getMutation(i);
3102         Durability tmpDur = getEffectiveDurability(m.getDurability());
3103         if (tmpDur.ordinal() > durability.ordinal()) {
3104           durability = tmpDur;
3105         }
3106         if (tmpDur == Durability.SKIP_WAL) {
3107           recordMutationWithoutWal(m.getFamilyCellMap());
3108           continue;
3109         }
3110 
3111         long nonceGroup = batchOp.getNonceGroup(i), nonce = batchOp.getNonce(i);
3112         // In replay, the batch may contain multiple nonces. If so, write WALEdit for each.
3113         // Given how nonces are originally written, these should be contiguous.
3114         // They don't have to be, it will still work, just write more WALEdits than needed.
3115         if (nonceGroup != currentNonceGroup || nonce != currentNonce) {
3116           if (walEdit.size() > 0) {
3117             assert isInReplay;
3118             if (!isInReplay) {
3119               throw new IOException("Multiple nonces per batch and not in replay");
3120             }
3121             // txid should always increase, so having the one from the last call is ok.
3122             // we use HLogKey here instead of WALKey directly to support legacy coprocessors.
3123             walKey = new ReplayHLogKey(this.getRegionInfo().getEncodedNameAsBytes(),
3124               this.htableDescriptor.getTableName(), now, m.getClusterIds(),
3125               currentNonceGroup, currentNonce, mvcc);
3126             txid = this.wal.append(this.htableDescriptor,  this.getRegionInfo(),  walKey,
3127               walEdit, true);
3128             walEdit = new WALEdit(isInReplay);
3129             walKey = null;
3130           }
3131           currentNonceGroup = nonceGroup;
3132           currentNonce = nonce;
3133         }
3134 
3135         // Add WAL edits by CP
3136         WALEdit fromCP = batchOp.walEditsFromCoprocessors[i];
3137         if (fromCP != null) {
3138           for (Cell cell : fromCP.getCells()) {
3139             walEdit.add(cell);
3140           }
3141         }
3142         addFamilyMapToWALEdit(familyMaps[i], walEdit);
3143       }
3144 
3145       // -------------------------
3146       // STEP 4. Append the final edit to WAL. Do not sync wal.
3147       // -------------------------
3148       Mutation mutation = batchOp.getMutation(firstIndex);
3149       if (isInReplay) {
3150         // use wal key from the original
3151         walKey = new ReplayHLogKey(this.getRegionInfo().getEncodedNameAsBytes(),
3152           this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now,
3153           mutation.getClusterIds(), currentNonceGroup, currentNonce, mvcc);
3154         long replaySeqId = batchOp.getReplaySequenceId();
3155         walKey.setOrigLogSeqNum(replaySeqId);
3156       }
3157       if (walEdit.size() > 0) {
3158         if (!isInReplay) {
3159         // we use HLogKey here instead of WALKey directly to support legacy coprocessors.
3160         walKey = new HLogKey(this.getRegionInfo().getEncodedNameAsBytes(),
3161             this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now,
3162             mutation.getClusterIds(), currentNonceGroup, currentNonce, mvcc);
3163         }
3164         txid = this.wal.append(this.htableDescriptor, this.getRegionInfo(), walKey, walEdit, true);
3165       }
3166       // ------------------------------------
3167       // Acquire the latest mvcc number
3168       // ----------------------------------
3169       if (walKey == null) {
3170         // If this is a skip wal operation just get the read point from mvcc
3171         walKey = this.appendEmptyEdit(this.wal);
3172       }
3173       if (!isInReplay) {
3174         writeEntry = walKey.getWriteEntry();
3175         mvccNum = writeEntry.getWriteNumber();
3176       } else {
3177         mvccNum = batchOp.getReplaySequenceId();
3178       }
3179 
3180       // ------------------------------------
3181       // STEP 5. Write back to memstore
3182       // Write to memstore. It is ok to write to memstore
3183       // first without syncing the WAL because we do not roll
3184       // forward the memstore MVCC. The MVCC will be moved up when
3185       // the complete operation is done. These changes are not yet
3186       // visible to scanners till we update the MVCC. The MVCC is
3187       // moved only when the sync is complete.
3188       // ----------------------------------
3189       long addedSize = 0;
3190       for (int i = firstIndex; i < lastIndexExclusive; i++) {
3191         if (batchOp.retCodeDetails[i].getOperationStatusCode()
3192             != OperationStatusCode.NOT_RUN) {
3193           continue;
3194         }
3195         doRollBackMemstore = true; // If we have a failure, we need to clean what we wrote
3196         addedSize += applyFamilyMapToMemstore(familyMaps[i], mvccNum, isInReplay);
3197       }
3198 
3199       // -------------------------------
3200       // STEP 6. Release row locks, etc.
3201       // -------------------------------
3202       if (locked) {
3203         this.updatesLock.readLock().unlock();
3204         locked = false;
3205       }
3206       releaseRowLocks(acquiredRowLocks);
3207 
3208       // -------------------------
3209       // STEP 7. Sync wal.
3210       // -------------------------
3211       if (txid != 0) {
3212         syncOrDefer(txid, durability);
3213       }
3214 
3215       doRollBackMemstore = false;
3216       // calling the post CP hook for batch mutation
3217       if (!isInReplay && coprocessorHost != null) {
3218         MiniBatchOperationInProgress<Mutation> miniBatchOp =
3219           new MiniBatchOperationInProgress<Mutation>(batchOp.getMutationsForCoprocs(),
3220           batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, lastIndexExclusive);
3221         coprocessorHost.postBatchMutate(miniBatchOp);
3222       }
3223 
3224       // ------------------------------------------------------------------
3225       // STEP 8. Advance mvcc. This will make this put visible to scanners and getters.
3226       // ------------------------------------------------------------------
3227       if (writeEntry != null) {
3228         mvcc.completeAndWait(writeEntry);
3229         writeEntry = null;
3230       } else if (isInReplay) {
3231         // ensure that the sequence id of the region is at least as big as orig log seq id
3232         mvcc.advanceTo(mvccNum);
3233       }
3234 
3235       for (int i = firstIndex; i < lastIndexExclusive; i ++) {
3236         if (batchOp.retCodeDetails[i] == OperationStatus.NOT_RUN) {
3237           batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
3238         }
3239       }
3240 
3241       // ------------------------------------
3242       // STEP 9. Run coprocessor post hooks. This should be done after the wal is
3243       // synced so that the coprocessor contract is adhered to.
3244       // ------------------------------------
3245       if (!isInReplay && coprocessorHost != null) {
3246         for (int i = firstIndex; i < lastIndexExclusive; i++) {
3247           // only for successful puts
3248           if (batchOp.retCodeDetails[i].getOperationStatusCode()
3249               != OperationStatusCode.SUCCESS) {
3250             continue;
3251           }
3252           Mutation m = batchOp.getMutation(i);
3253           if (m instanceof Put) {
3254             coprocessorHost.postPut((Put) m, walEdit, m.getDurability());
3255           } else {
3256             coprocessorHost.postDelete((Delete) m, walEdit, m.getDurability());
3257           }
3258         }
3259       }
3260 
3261       success = true;
3262       return addedSize;
3263     } finally {
3264       // if the wal sync was unsuccessful, remove keys from memstore
3265       if (doRollBackMemstore) {
3266         for (int j = 0; j < familyMaps.length; j++) {
3267           for(List<Cell> cells:familyMaps[j].values()) {
3268             rollbackMemstore(cells);
3269           }
3270         }
3271         if (writeEntry != null) mvcc.complete(writeEntry);
3272       } else if (writeEntry != null) {
3273         mvcc.completeAndWait(writeEntry);
3274       }
3275 
3276       if (locked) {
3277         this.updatesLock.readLock().unlock();
3278       }
3279       releaseRowLocks(acquiredRowLocks);
3280 
3281       // See if the column families were consistent through the whole thing.
3282       // if they were then keep them. If they were not then pass a null.
3283       // null will be treated as unknown.
3284       // Total time taken might be involving Puts and Deletes.
3285       // Split the time for puts and deletes based on the total number of Puts and Deletes.
3286 
3287       if (noOfPuts > 0) {
3288         // There were some Puts in the batch.
3289         if (this.metricsRegion != null) {
3290           this.metricsRegion.updatePut();
3291         }
3292       }
3293       if (noOfDeletes > 0) {
3294         // There were some Deletes in the batch.
3295         if (this.metricsRegion != null) {
3296           this.metricsRegion.updateDelete();
3297         }
3298       }
3299       if (!success) {
3300         for (int i = firstIndex; i < lastIndexExclusive; i++) {
3301           if (batchOp.retCodeDetails[i].getOperationStatusCode() == OperationStatusCode.NOT_RUN) {
3302             batchOp.retCodeDetails[i] = OperationStatus.FAILURE;
3303           }
3304         }
3305       }
3306       if (coprocessorHost != null && !batchOp.isInReplay()) {
3307         // call the coprocessor hook to do any finalization steps
3308         // after the put is done
3309         MiniBatchOperationInProgress<Mutation> miniBatchOp =
3310             new MiniBatchOperationInProgress<Mutation>(batchOp.getMutationsForCoprocs(),
3311                 batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex,
3312                 lastIndexExclusive);
3313         coprocessorHost.postBatchMutateIndispensably(miniBatchOp, success);
3314       }
3315 
3316       batchOp.nextIndexToProcess = lastIndexExclusive;
3317     }
3318   }
3319 
3320   /**
3321    * Returns effective durability from the passed durability and
3322    * the table descriptor.
3323    */
3324   protected Durability getEffectiveDurability(Durability d) {
3325     return d == Durability.USE_DEFAULT ? this.durability : d;
3326   }
3327 
3328   //TODO, Think that gets/puts and deletes should be refactored a bit so that
3329   //the getting of the lock happens before, so that you would just pass it into
3330   //the methods. So in the case of checkAndMutate you could just do lockRow,
3331   //get, put, unlockRow or something
3332 
3333   @Override
3334   public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
3335       CompareOp compareOp, ByteArrayComparable comparator, Mutation w,
3336       boolean writeToWAL)
3337   throws IOException{
3338     checkReadOnly();
3339     //TODO, add check for value length or maybe even better move this to the
3340     //client if this becomes a global setting
3341     checkResources();
3342     boolean isPut = w instanceof Put;
3343     if (!isPut && !(w instanceof Delete))
3344       throw new org.apache.hadoop.hbase.DoNotRetryIOException("Action must " +
3345           "be Put or Delete");
3346     if (!Bytes.equals(row, w.getRow())) {
3347       throw new org.apache.hadoop.hbase.DoNotRetryIOException("Action's " +
3348           "getRow must match the passed row");
3349     }
3350 
3351     startRegionOperation();
3352     try {
3353       Get get = new Get(row);
3354       checkFamily(family);
3355       get.addColumn(family, qualifier);
3356 
3357       // Lock row - note that doBatchMutate will relock this row if called
3358       RowLock rowLock = getRowLock(get.getRow());
3359       // wait for all previous transactions to complete (with lock held)
3360       mvcc.await();
3361       try {
3362         if (this.getCoprocessorHost() != null) {
3363           Boolean processed = null;
3364           if (w instanceof Put) {
3365             processed = this.getCoprocessorHost().preCheckAndPutAfterRowLock(row, family,
3366                 qualifier, compareOp, comparator, (Put) w);
3367           } else if (w instanceof Delete) {
3368             processed = this.getCoprocessorHost().preCheckAndDeleteAfterRowLock(row, family,
3369                 qualifier, compareOp, comparator, (Delete) w);
3370           }
3371           if (processed != null) {
3372             return processed;
3373           }
3374         }
3375         List<Cell> result = get(get, false);
3376 
3377         boolean valueIsNull = comparator.getValue() == null ||
3378           comparator.getValue().length == 0;
3379         boolean matches = false;
3380         long cellTs = 0;
3381         if (result.size() == 0 && valueIsNull) {
3382           matches = true;
3383         } else if (result.size() > 0 && result.get(0).getValueLength() == 0 &&
3384             valueIsNull) {
3385           matches = true;
3386           cellTs = result.get(0).getTimestamp();
3387         } else if (result.size() == 1 && !valueIsNull) {
3388           Cell kv = result.get(0);
3389           cellTs = kv.getTimestamp();
3390           int compareResult = comparator.compareTo(kv.getValueArray(),
3391               kv.getValueOffset(), kv.getValueLength());
3392           switch (compareOp) {
3393           case LESS:
3394             matches = compareResult < 0;
3395             break;
3396           case LESS_OR_EQUAL:
3397             matches = compareResult <= 0;
3398             break;
3399           case EQUAL:
3400             matches = compareResult == 0;
3401             break;
3402           case NOT_EQUAL:
3403             matches = compareResult != 0;
3404             break;
3405           case GREATER_OR_EQUAL:
3406             matches = compareResult >= 0;
3407             break;
3408           case GREATER:
3409             matches = compareResult > 0;
3410             break;
3411           default:
3412             throw new RuntimeException("Unknown Compare op " + compareOp.name());
3413           }
3414         }
3415         //If matches put the new put or delete the new delete
3416         if (matches) {
3417           // We have acquired the row lock already. If the system clock is NOT monotonically
3418           // non-decreasing (see HBASE-14070) we should make sure that the mutation has a
3419           // larger timestamp than what was observed via Get. doBatchMutate already does this, but
3420           // there is no way to pass the cellTs. See HBASE-14054.
3421           long now = EnvironmentEdgeManager.currentTime();
3422           long ts = Math.max(now, cellTs); // ensure write is not eclipsed
3423           byte[] byteTs = Bytes.toBytes(ts);
3424 
3425           if (w instanceof Put) {
3426             updateCellTimestamps(w.getFamilyCellMap().values(), byteTs);
3427           }
3428           // else delete is not needed since it already does a second get, and sets the timestamp
3429           // from get (see prepareDeleteTimestamps).
3430 
3431           // All edits for the given row (across all column families) must
3432           // happen atomically.
3433           doBatchMutate(w);
3434           this.checkAndMutateChecksPassed.increment();
3435           return true;
3436         }
3437         this.checkAndMutateChecksFailed.increment();
3438         return false;
3439       } finally {
3440         rowLock.release();
3441       }
3442     } finally {
3443       closeRegionOperation();
3444     }
3445   }
3446 
3447   //TODO, Think that gets/puts and deletes should be refactored a bit so that
3448   //the getting of the lock happens before, so that you would just pass it into
3449   //the methods. So in the case of checkAndMutate you could just do lockRow,
3450   //get, put, unlockRow or something
3451 
3452   @Override
3453   public boolean checkAndRowMutate(byte [] row, byte [] family, byte [] qualifier,
3454       CompareOp compareOp, ByteArrayComparable comparator, RowMutations rm,
3455       boolean writeToWAL) throws IOException {
3456     checkReadOnly();
3457     //TODO, add check for value length or maybe even better move this to the
3458     //client if this becomes a global setting
3459     checkResources();
3460 
3461     startRegionOperation();
3462     try {
3463       Get get = new Get(row);
3464       checkFamily(family);
3465       get.addColumn(family, qualifier);
3466 
3467       // Lock row - note that doBatchMutate will relock this row if called
3468       RowLock rowLock = getRowLock(get.getRow());
3469       // wait for all previous transactions to complete (with lock held)
3470       mvcc.await();
3471       try {
3472         List<Cell> result = get(get, false);
3473 
3474         boolean valueIsNull = comparator.getValue() == null ||
3475             comparator.getValue().length == 0;
3476         boolean matches = false;
3477         long cellTs = 0;
3478         if (result.size() == 0 && valueIsNull) {
3479           matches = true;
3480         } else if (result.size() > 0 && result.get(0).getValueLength() == 0 &&
3481             valueIsNull) {
3482           matches = true;
3483           cellTs = result.get(0).getTimestamp();
3484         } else if (result.size() == 1 && !valueIsNull) {
3485           Cell kv = result.get(0);
3486           cellTs = kv.getTimestamp();
3487           int compareResult = comparator.compareTo(kv.getValueArray(),
3488               kv.getValueOffset(), kv.getValueLength());
3489           switch (compareOp) {
3490           case LESS:
3491             matches = compareResult < 0;
3492             break;
3493           case LESS_OR_EQUAL:
3494             matches = compareResult <= 0;
3495             break;
3496           case EQUAL:
3497             matches = compareResult == 0;
3498             break;
3499           case NOT_EQUAL:
3500             matches = compareResult != 0;
3501             break;
3502           case GREATER_OR_EQUAL:
3503             matches = compareResult >= 0;
3504             break;
3505           case GREATER:
3506             matches = compareResult > 0;
3507             break;
3508           default:
3509             throw new RuntimeException("Unknown Compare op " + compareOp.name());
3510           }
3511         }
3512         //If matches put the new put or delete the new delete
3513         if (matches) {
3514           // We have acquired the row lock already. If the system clock is NOT monotonically
3515           // non-decreasing (see HBASE-14070) we should make sure that the mutation has a
3516           // larger timestamp than what was observed via Get. doBatchMutate already does this, but
3517           // there is no way to pass the cellTs. See HBASE-14054.
3518           long now = EnvironmentEdgeManager.currentTime();
3519           long ts = Math.max(now, cellTs); // ensure write is not eclipsed
3520           byte[] byteTs = Bytes.toBytes(ts);
3521 
3522           for (Mutation w : rm.getMutations()) {
3523             if (w instanceof Put) {
3524               updateCellTimestamps(w.getFamilyCellMap().values(), byteTs);
3525             }
3526             // else delete is not needed since it already does a second get, and sets the timestamp
3527             // from get (see prepareDeleteTimestamps).
3528           }
3529 
3530           // All edits for the given row (across all column families) must
3531           // happen atomically.
3532           mutateRow(rm);
3533           this.checkAndMutateChecksPassed.increment();
3534           return true;
3535         }
3536         this.checkAndMutateChecksFailed.increment();
3537         return false;
3538       } finally {
3539         rowLock.release();
3540       }
3541     } finally {
3542       closeRegionOperation();
3543     }
3544   }
3545 
3546   private void doBatchMutate(Mutation mutation) throws IOException {
3547     // Currently this is only called for puts and deletes, so no nonces.
3548     OperationStatus[] batchMutate = this.batchMutate(new Mutation[]{mutation});
3549     if (batchMutate[0].getOperationStatusCode().equals(OperationStatusCode.SANITY_CHECK_FAILURE)) {
3550       throw new FailedSanityCheckException(batchMutate[0].getExceptionMsg());
3551     } else if (batchMutate[0].getOperationStatusCode().equals(OperationStatusCode.BAD_FAMILY)) {
3552       throw new NoSuchColumnFamilyException(batchMutate[0].getExceptionMsg());
3553     }
3554   }
3555 
3556   /**
3557    * Complete taking the snapshot on the region. Writes the region info and adds references to the
3558    * working snapshot directory.
3559    *
3560    * TODO for api consistency, consider adding another version with no {@link ForeignExceptionSnare}
3561    * arg.  (In the future other cancellable HRegion methods could eventually add a
3562    * {@link ForeignExceptionSnare}, or we could do something fancier).
3563    *
3564    * @param desc snapshot description object
3565    * @param exnSnare ForeignExceptionSnare that captures external exceptions in case we need to
3566    *   bail out.  This is allowed to be null and will just be ignored in that case.
3567    * @throws IOException if there is an external or internal error causing the snapshot to fail
3568    */
3569   public void addRegionToSnapshot(SnapshotDescription desc,
3570       ForeignExceptionSnare exnSnare) throws IOException {
3571     Path rootDir = FSUtils.getRootDir(conf);
3572     Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
3573 
3574     SnapshotManifest manifest = SnapshotManifest.create(conf, getFilesystem(),
3575             snapshotDir, desc, exnSnare);
3576     manifest.addRegion(this);
3577 
3578     // The regionserver holding the first region of the table is responsible for taking the
3579     // manifest of the mob dir.
3580     if (!Bytes.equals(getRegionInfo().getStartKey(), HConstants.EMPTY_START_ROW))
3581       return;
3582 
3583     // if any cf's have is mob enabled, add the "mob region" to the manifest.
3584     List<Store> stores = getStores();
3585     for (Store store : stores) {
3586       boolean hasMobStore = store.getFamily().isMobEnabled();
3587       if (hasMobStore) {
3588         // use the .mob as the start key and 0 as the regionid
3589         HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(this.getTableDesc().getTableName());
3590         mobRegionInfo.setOffline(true);
3591         manifest.addMobRegion(mobRegionInfo, this.getTableDesc().getColumnFamilies());
3592         return;
3593       }
3594     }
3595   }
3596 
3597   @Override
3598   public void updateCellTimestamps(final Iterable<List<Cell>> cellItr, final byte[] now)
3599       throws IOException {
3600     for (List<Cell> cells: cellItr) {
3601       if (cells == null) continue;
3602       assert cells instanceof RandomAccess;
3603       int listSize = cells.size();
3604       for (int i = 0; i < listSize; i++) {
3605         CellUtil.updateLatestStamp(cells.get(i), now, 0);
3606       }
3607     }
3608   }
3609 
3610   /**
3611    * Possibly rewrite incoming cell tags.
3612    */
3613   void rewriteCellTags(Map<byte[], List<Cell>> familyMap, final Mutation m) {
3614     // Check if we have any work to do and early out otherwise
3615     // Update these checks as more logic is added here
3616 
3617     if (m.getTTL() == Long.MAX_VALUE) {
3618       return;
3619     }
3620 
3621     // From this point we know we have some work to do
3622 
3623     for (Map.Entry<byte[], List<Cell>> e: familyMap.entrySet()) {
3624       List<Cell> cells = e.getValue();
3625       assert cells instanceof RandomAccess;
3626       int listSize = cells.size();
3627       for (int i = 0; i < listSize; i++) {
3628         Cell cell = cells.get(i);
3629         List<Tag> newTags = Tag.carryForwardTags(null, cell);
3630         newTags = carryForwardTTLTag(newTags, m);
3631 
3632         // Rewrite the cell with the updated set of tags
3633         cells.set(i, new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
3634           cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
3635           cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
3636           cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()),
3637           cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(),
3638           newTags));
3639       }
3640     }
3641   }
3642 
3643   /*
3644    * Check if resources to support an update.
3645    *
3646    * We throw RegionTooBusyException if above memstore limit
3647    * and expect client to retry using some kind of backoff
3648   */
3649   private void checkResources() throws RegionTooBusyException {
3650     // If catalog region, do not impose resource constraints or block updates.
3651     if (this.getRegionInfo().isMetaRegion()) return;
3652 
3653     if (this.memstoreSize.get() > this.blockingMemStoreSize) {
3654       blockedRequestsCount.increment();
3655       requestFlush();
3656       throw new RegionTooBusyException("Above memstore limit, " +
3657           "regionName=" + (this.getRegionInfo() == null ? "unknown" :
3658           this.getRegionInfo().getRegionNameAsString()) +
3659           ", server=" + (this.getRegionServerServices() == null ? "unknown" :
3660           this.getRegionServerServices().getServerName()) +
3661           ", memstoreSize=" + memstoreSize.get() +
3662           ", blockingMemStoreSize=" + blockingMemStoreSize);
3663     }
3664   }
3665 
3666   /**
3667    * @throws IOException Throws exception if region is in read-only mode.
3668    */
3669   protected void checkReadOnly() throws IOException {
3670     if (isReadOnly()) {
3671       throw new DoNotRetryIOException("region is read only");
3672     }
3673   }
3674 
3675   protected void checkReadsEnabled() throws IOException {
3676     if (!this.writestate.readsEnabled) {
3677       throw new IOException(getRegionInfo().getEncodedName()
3678         + ": The region's reads are disabled. Cannot serve the request");
3679     }
3680   }
3681 
3682   public void setReadsEnabled(boolean readsEnabled) {
3683    if (readsEnabled && !this.writestate.readsEnabled) {
3684      LOG.info(getRegionInfo().getEncodedName() + " : Enabling reads for region.");
3685     }
3686     this.writestate.setReadsEnabled(readsEnabled);
3687   }
3688 
3689   /**
3690    * Add updates first to the wal and then add values to memstore.
3691    * Warning: Assumption is caller has lock on passed in row.
3692    * @param edits Cell updates by column
3693    * @throws IOException
3694    */
3695   private void put(final byte [] row, byte [] family, List<Cell> edits)
3696   throws IOException {
3697     NavigableMap<byte[], List<Cell>> familyMap;
3698     familyMap = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
3699 
3700     familyMap.put(family, edits);
3701     Put p = new Put(row);
3702     p.setFamilyCellMap(familyMap);
3703     doBatchMutate(p);
3704   }
3705 
3706   /**
3707    * Atomically apply the given map of family->edits to the memstore.
3708    * This handles the consistency control on its own, but the caller
3709    * should already have locked updatesLock.readLock(). This also does
3710    * <b>not</b> check the families for validity.
3711    *
3712    * @param familyMap Map of kvs per family
3713    * @param localizedWriteEntry The WriteEntry of the MVCC for this transaction.
3714    *        If null, then this method internally creates a mvcc transaction.
3715    * @param output newly added KVs into memstore
3716    * @param isInReplay true when adding replayed KVs into memstore
3717    * @return the additional memory usage of the memstore caused by the
3718    * new entries.
3719    * @throws IOException
3720    */
3721   private long applyFamilyMapToMemstore(Map<byte[], List<Cell>> familyMap,
3722     long mvccNum, boolean isInReplay) throws IOException {
3723     long size = 0;
3724 
3725     for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {
3726       byte[] family = e.getKey();
3727       List<Cell> cells = e.getValue();
3728       assert cells instanceof RandomAccess;
3729       Store store = getStore(family);
3730       int listSize = cells.size();
3731       for (int i=0; i < listSize; i++) {
3732         Cell cell = cells.get(i);
3733         if (cell.getSequenceId() == 0 || isInReplay) {
3734           CellUtil.setSequenceId(cell, mvccNum);
3735         }
3736         size += store.add(cell);
3737       }
3738     }
3739 
3740      return size;
3741    }
3742 
3743   /**
3744    * Remove all the keys listed in the map from the memstore. This method is
3745    * called when a Put/Delete has updated memstore but subsequently fails to update
3746    * the wal. This method is then invoked to rollback the memstore.
3747    */
3748   private void rollbackMemstore(List<Cell> memstoreCells) {
3749     int kvsRolledback = 0;
3750 
3751     for (Cell cell : memstoreCells) {
3752       byte[] family = CellUtil.cloneFamily(cell);
3753       Store store = getStore(family);
3754       store.rollback(cell);
3755       kvsRolledback++;
3756     }
3757     LOG.debug("rollbackMemstore rolled back " + kvsRolledback);
3758   }
3759 
3760   @Override
3761   public void checkFamilies(Collection<byte[]> families) throws NoSuchColumnFamilyException {
3762     for (byte[] family : families) {
3763       checkFamily(family);
3764     }
3765   }
3766 
3767   /**
3768    * During replay, there could exist column families which are removed between region server
3769    * failure and replay
3770    */
3771   private void removeNonExistentColumnFamilyForReplay(
3772       final Map<byte[], List<Cell>> familyMap) {
3773     List<byte[]> nonExistentList = null;
3774     for (byte[] family : familyMap.keySet()) {
3775       if (!this.htableDescriptor.hasFamily(family)) {
3776         if (nonExistentList == null) {
3777           nonExistentList = new ArrayList<byte[]>();
3778         }
3779         nonExistentList.add(family);
3780       }
3781     }
3782     if (nonExistentList != null) {
3783       for (byte[] family : nonExistentList) {
3784         // Perhaps schema was changed between crash and replay
3785         LOG.info("No family for " + Bytes.toString(family) + " omit from reply.");
3786         familyMap.remove(family);
3787       }
3788     }
3789   }
3790 
3791   @Override
3792   public void checkTimestamps(final Map<byte[], List<Cell>> familyMap, long now)
3793       throws FailedSanityCheckException {
3794     if (timestampSlop == HConstants.LATEST_TIMESTAMP) {
3795       return;
3796     }
3797     long maxTs = now + timestampSlop;
3798     for (List<Cell> kvs : familyMap.values()) {
3799       assert kvs instanceof RandomAccess;
3800       int listSize  = kvs.size();
3801       for (int i=0; i < listSize; i++) {
3802         Cell cell = kvs.get(i);
3803         // see if the user-side TS is out of range. latest = server-side
3804         long ts = cell.getTimestamp();
3805         if (ts != HConstants.LATEST_TIMESTAMP && ts > maxTs) {
3806           throw new FailedSanityCheckException("Timestamp for KV out of range "
3807               + cell + " (too.new=" + timestampSlop + ")");
3808         }
3809       }
3810     }
3811   }
3812 
3813   /**
3814    * Append the given map of family->edits to a WALEdit data structure.
3815    * This does not write to the WAL itself.
3816    * @param familyMap map of family->edits
3817    * @param walEdit the destination entry to append into
3818    */
3819   private void addFamilyMapToWALEdit(Map<byte[], List<Cell>> familyMap,
3820       WALEdit walEdit) {
3821     for (List<Cell> edits : familyMap.values()) {
3822       assert edits instanceof RandomAccess;
3823       int listSize = edits.size();
3824       for (int i=0; i < listSize; i++) {
3825         Cell cell = edits.get(i);
3826         walEdit.add(cell);
3827       }
3828     }
3829   }
3830 
3831   private void requestFlush() {
3832     if (this.rsServices == null) {
3833       return;
3834     }
3835     synchronized (writestate) {
3836       if (this.writestate.isFlushRequested()) {
3837         return;
3838       }
3839       writestate.flushRequested = true;
3840     }
3841     // Make request outside of synchronize block; HBASE-818.
3842     this.rsServices.getFlushRequester().requestFlush(this, false);
3843     if (LOG.isDebugEnabled()) {
3844       LOG.debug("Flush requested on " + this.getRegionInfo().getEncodedName());
3845     }
3846   }
3847 
3848   /*
3849    * @param size
3850    * @return True if size is over the flush threshold
3851    */
3852   private boolean isFlushSize(final long size) {
3853     return size > this.memstoreFlushSize;
3854   }
3855 
3856   /**
3857    * Read the edits put under this region by wal splitting process.  Put
3858    * the recovered edits back up into this region.
3859    *
3860    * <p>We can ignore any wal message that has a sequence ID that's equal to or
3861    * lower than minSeqId.  (Because we know such messages are already
3862    * reflected in the HFiles.)
3863    *
3864    * <p>While this is running we are putting pressure on memory yet we are
3865    * outside of our usual accounting because we are not yet an onlined region
3866    * (this stuff is being run as part of Region initialization).  This means
3867    * that if we're up against global memory limits, we'll not be flagged to flush
3868    * because we are not online. We can't be flushed by usual mechanisms anyways;
3869    * we're not yet online so our relative sequenceids are not yet aligned with
3870    * WAL sequenceids -- not till we come up online, post processing of split
3871    * edits.
3872    *
3873    * <p>But to help relieve memory pressure, at least manage our own heap size
3874    * flushing if are in excess of per-region limits.  Flushing, though, we have
3875    * to be careful and avoid using the regionserver/wal sequenceid.  Its running
3876    * on a different line to whats going on in here in this region context so if we
3877    * crashed replaying these edits, but in the midst had a flush that used the
3878    * regionserver wal with a sequenceid in excess of whats going on in here
3879    * in this region and with its split editlogs, then we could miss edits the
3880    * next time we go to recover. So, we have to flush inline, using seqids that
3881    * make sense in a this single region context only -- until we online.
3882    *
3883    * @param maxSeqIdInStores Any edit found in split editlogs needs to be in excess of
3884    * the maxSeqId for the store to be applied, else its skipped.
3885    * @return the sequence id of the last edit added to this region out of the
3886    * recovered edits log or <code>minSeqId</code> if nothing added from editlogs.
3887    * @throws IOException
3888    */
3889   protected long replayRecoveredEditsIfAny(final Path regiondir,
3890       Map<byte[], Long> maxSeqIdInStores,
3891       final CancelableProgressable reporter, final MonitoredTask status)
3892       throws IOException {
3893     long minSeqIdForTheRegion = -1;
3894     for (Long maxSeqIdInStore : maxSeqIdInStores.values()) {
3895       if (maxSeqIdInStore < minSeqIdForTheRegion || minSeqIdForTheRegion == -1) {
3896         minSeqIdForTheRegion = maxSeqIdInStore;
3897       }
3898     }
3899     long seqid = minSeqIdForTheRegion;
3900 
3901     FileSystem fs = this.fs.getFileSystem();
3902     NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regiondir);
3903     if (LOG.isDebugEnabled()) {
3904       LOG.debug("Found " + (files == null ? 0 : files.size())
3905         + " recovered edits file(s) under " + regiondir);
3906     }
3907 
3908     if (files == null || files.isEmpty()) return seqid;
3909 
3910     for (Path edits: files) {
3911       if (edits == null || !fs.exists(edits)) {
3912         LOG.warn("Null or non-existent edits file: " + edits);
3913         continue;
3914       }
3915       if (isZeroLengthThenDelete(fs, edits)) continue;
3916 
3917       long maxSeqId;
3918       String fileName = edits.getName();
3919       maxSeqId = Math.abs(Long.parseLong(fileName));
3920       if (maxSeqId <= minSeqIdForTheRegion) {
3921         if (LOG.isDebugEnabled()) {
3922           String msg = "Maximum sequenceid for this wal is " + maxSeqId
3923             + " and minimum sequenceid for the region is " + minSeqIdForTheRegion
3924             + ", skipped the whole file, path=" + edits;
3925           LOG.debug(msg);
3926         }
3927         continue;
3928       }
3929 
3930       try {
3931         // replay the edits. Replay can return -1 if everything is skipped, only update
3932         // if seqId is greater
3933         seqid = Math.max(seqid, replayRecoveredEdits(edits, maxSeqIdInStores, reporter));
3934       } catch (IOException e) {
3935         boolean skipErrors = conf.getBoolean(
3936             HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS,
3937             conf.getBoolean(
3938                 "hbase.skip.errors",
3939                 HConstants.DEFAULT_HREGION_EDITS_REPLAY_SKIP_ERRORS));
3940         if (conf.get("hbase.skip.errors") != null) {
3941           LOG.warn(
3942               "The property 'hbase.skip.errors' has been deprecated. Please use " +
3943               HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS + " instead.");
3944         }
3945         if (skipErrors) {
3946           Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
3947           LOG.error(HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS
3948               + "=true so continuing. Renamed " + edits +
3949               " as " + p, e);
3950         } else {
3951           throw e;
3952         }
3953       }
3954     }
3955     // The edits size added into rsAccounting during this replaying will not
3956     // be required any more. So just clear it.
3957     if (this.rsAccounting != null) {
3958       this.rsAccounting.clearRegionReplayEditsSize(getRegionInfo().getRegionName());
3959     }
3960     if (seqid > minSeqIdForTheRegion) {
3961       // Then we added some edits to memory. Flush and cleanup split edit files.
3962       internalFlushcache(null, seqid, stores.values(), status, false);
3963     }
3964     // Now delete the content of recovered edits.  We're done w/ them.
3965     if (files.size() > 0 && this.conf.getBoolean("hbase.region.archive.recovered.edits", false)) {
3966       // For debugging data loss issues!
3967       // If this flag is set, make use of the hfile archiving by making recovered.edits a fake
3968       // column family. Have to fake out file type too by casting our recovered.edits as storefiles
3969       String fakeFamilyName = WALSplitter.getRegionDirRecoveredEditsDir(regiondir).getName();
3970       Set<StoreFile> fakeStoreFiles = new HashSet<StoreFile>(files.size());
3971       for (Path file: files) {
3972         fakeStoreFiles.add(new StoreFile(getRegionFileSystem().getFileSystem(), file, this.conf,
3973           null, null));
3974       }
3975       getRegionFileSystem().removeStoreFiles(fakeFamilyName, fakeStoreFiles);
3976     } else {
3977       for (Path file: files) {
3978         if (!fs.delete(file, false)) {
3979           LOG.error("Failed delete of " + file);
3980         } else {
3981           LOG.debug("Deleted recovered.edits file=" + file);
3982         }
3983       }
3984     }
3985     return seqid;
3986   }
3987 
3988   /*
3989    * @param edits File of recovered edits.
3990    * @param maxSeqIdInStores Maximum sequenceid found in each store.  Edits in wal
3991    * must be larger than this to be replayed for each store.
3992    * @param reporter
3993    * @return the sequence id of the last edit added to this region out of the
3994    * recovered edits log or <code>minSeqId</code> if nothing added from editlogs.
3995    * @throws IOException
3996    */
3997   private long replayRecoveredEdits(final Path edits,
3998       Map<byte[], Long> maxSeqIdInStores, final CancelableProgressable reporter)
3999     throws IOException {
4000     String msg = "Replaying edits from " + edits;
4001     LOG.info(msg);
4002     MonitoredTask status = TaskMonitor.get().createStatus(msg);
4003     FileSystem fs = this.fs.getFileSystem();
4004 
4005     status.setStatus("Opening recovered edits");
4006     WAL.Reader reader = null;
4007     try {
4008       reader = WALFactory.createReader(fs, edits, conf);
4009       long currentEditSeqId = -1;
4010       long currentReplaySeqId = -1;
4011       long firstSeqIdInLog = -1;
4012       long skippedEdits = 0;
4013       long editsCount = 0;
4014       long intervalEdits = 0;
4015       WAL.Entry entry;
4016       Store store = null;
4017       boolean reported_once = false;
4018       ServerNonceManager ng = this.rsServices == null ? null : this.rsServices.getNonceManager();
4019 
4020       try {
4021         // How many edits seen before we check elapsed time
4022         int interval = this.conf.getInt("hbase.hstore.report.interval.edits", 2000);
4023         // How often to send a progress report (default 1/2 master timeout)
4024         int period = this.conf.getInt("hbase.hstore.report.period", 300000);
4025         long lastReport = EnvironmentEdgeManager.currentTime();
4026 
4027         while ((entry = reader.next()) != null) {
4028           WALKey key = entry.getKey();
4029           WALEdit val = entry.getEdit();
4030 
4031           if (ng != null) { // some test, or nonces disabled
4032             ng.reportOperationFromWal(key.getNonceGroup(), key.getNonce(), key.getWriteTime());
4033           }
4034 
4035           if (reporter != null) {
4036             intervalEdits += val.size();
4037             if (intervalEdits >= interval) {
4038               // Number of edits interval reached
4039               intervalEdits = 0;
4040               long cur = EnvironmentEdgeManager.currentTime();
4041               if (lastReport + period <= cur) {
4042                 status.setStatus("Replaying edits..." +
4043                     " skipped=" + skippedEdits +
4044                     " edits=" + editsCount);
4045                 // Timeout reached
4046                 if(!reporter.progress()) {
4047                   msg = "Progressable reporter failed, stopping replay";
4048                   LOG.warn(msg);
4049                   status.abort(msg);
4050                   throw new IOException(msg);
4051                 }
4052                 reported_once = true;
4053                 lastReport = cur;
4054               }
4055             }
4056           }
4057 
4058           if (firstSeqIdInLog == -1) {
4059             firstSeqIdInLog = key.getLogSeqNum();
4060           }
4061           if (currentEditSeqId > key.getLogSeqNum()) {
4062             // when this condition is true, it means we have a serious defect because we need to
4063             // maintain increasing SeqId for WAL edits per region
4064             LOG.error(getRegionInfo().getEncodedName() + " : "
4065                  + "Found decreasing SeqId. PreId=" + currentEditSeqId + " key=" + key
4066                 + "; edit=" + val);
4067           } else {
4068             currentEditSeqId = key.getLogSeqNum();
4069           }
4070           currentReplaySeqId = (key.getOrigLogSeqNum() > 0) ?
4071             key.getOrigLogSeqNum() : currentEditSeqId;
4072 
4073           // Start coprocessor replay here. The coprocessor is for each WALEdit
4074           // instead of a KeyValue.
4075           if (coprocessorHost != null) {
4076             status.setStatus("Running pre-WAL-restore hook in coprocessors");
4077             if (coprocessorHost.preWALRestore(this.getRegionInfo(), key, val)) {
4078               // if bypass this wal entry, ignore it ...
4079               continue;
4080             }
4081           }
4082           boolean checkRowWithinBoundary = false;
4083           // Check this edit is for this region.
4084           if (!Bytes.equals(key.getEncodedRegionName(),
4085               this.getRegionInfo().getEncodedNameAsBytes())) {
4086             checkRowWithinBoundary = true;
4087           }
4088 
4089           boolean flush = false;
4090           for (Cell cell: val.getCells()) {
4091             // Check this edit is for me. Also, guard against writing the special
4092             // METACOLUMN info such as HBASE::CACHEFLUSH entries
4093             if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) {
4094               // if region names don't match, skipp replaying compaction marker
4095               if (!checkRowWithinBoundary) {
4096                 //this is a special edit, we should handle it
4097                 CompactionDescriptor compaction = WALEdit.getCompaction(cell);
4098                 if (compaction != null) {
4099                   //replay the compaction
4100                   replayWALCompactionMarker(compaction, false, true, Long.MAX_VALUE);
4101                 }
4102               }
4103               skippedEdits++;
4104               continue;
4105             }
4106             // Figure which store the edit is meant for.
4107             if (store == null || !CellUtil.matchingFamily(cell, store.getFamily().getName())) {
4108               store = getStore(cell);
4109             }
4110             if (store == null) {
4111               // This should never happen.  Perhaps schema was changed between
4112               // crash and redeploy?
4113               LOG.warn("No family for " + cell);
4114               skippedEdits++;
4115               continue;
4116             }
4117             if (checkRowWithinBoundary && !rowIsInRange(this.getRegionInfo(),
4118               cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())) {
4119               LOG.warn("Row of " + cell + " is not within region boundary");
4120               skippedEdits++;
4121               continue;
4122             }
4123             // Now, figure if we should skip this edit.
4124             if (key.getLogSeqNum() <= maxSeqIdInStores.get(store.getFamily()
4125                 .getName())) {
4126               skippedEdits++;
4127               continue;
4128             }
4129             CellUtil.setSequenceId(cell, currentReplaySeqId);
4130 
4131             // Once we are over the limit, restoreEdit will keep returning true to
4132             // flush -- but don't flush until we've played all the kvs that make up
4133             // the WALEdit.
4134             flush |= restoreEdit(store, cell);
4135             editsCount++;
4136           }
4137           if (flush) {
4138             internalFlushcache(null, currentEditSeqId, stores.values(), status, false);
4139           }
4140 
4141           if (coprocessorHost != null) {
4142             coprocessorHost.postWALRestore(this.getRegionInfo(), key, val);
4143           }
4144         }
4145       } catch (EOFException eof) {
4146         Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
4147         msg = "Encountered EOF. Most likely due to Master failure during " +
4148             "wal splitting, so we have this data in another edit.  " +
4149             "Continuing, but renaming " + edits + " as " + p;
4150         LOG.warn(msg, eof);
4151         status.abort(msg);
4152       } catch (IOException ioe) {
4153         // If the IOE resulted from bad file format,
4154         // then this problem is idempotent and retrying won't help
4155         if (ioe.getCause() instanceof ParseException) {
4156           Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
4157           msg = "File corruption encountered!  " +
4158               "Continuing, but renaming " + edits + " as " + p;
4159           LOG.warn(msg, ioe);
4160           status.setStatus(msg);
4161         } else {
4162           status.abort(StringUtils.stringifyException(ioe));
4163           // other IO errors may be transient (bad network connection,
4164           // checksum exception on one datanode, etc).  throw & retry
4165           throw ioe;
4166         }
4167       }
4168       if (reporter != null && !reported_once) {
4169         reporter.progress();
4170       }
4171       msg = "Applied " + editsCount + ", skipped " + skippedEdits +
4172         ", firstSequenceIdInLog=" + firstSeqIdInLog +
4173         ", maxSequenceIdInLog=" + currentEditSeqId + ", path=" + edits;
4174       status.markComplete(msg);
4175       LOG.debug(msg);
4176       return currentEditSeqId;
4177     } finally {
4178       status.cleanup();
4179       if (reader != null) {
4180          reader.close();
4181       }
4182     }
4183   }
4184 
4185   /**
4186    * Call to complete a compaction. Its for the case where we find in the WAL a compaction
4187    * that was not finished.  We could find one recovering a WAL after a regionserver crash.
4188    * See HBASE-2331.
4189    */
4190   void replayWALCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles,
4191       boolean removeFiles, long replaySeqId)
4192       throws IOException {
4193     try {
4194       checkTargetRegion(compaction.getEncodedRegionName().toByteArray(),
4195         "Compaction marker from WAL ", compaction);
4196     } catch (WrongRegionException wre) {
4197       if (RegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {
4198         // skip the compaction marker since it is not for this region
4199         return;
4200       }
4201       throw wre;
4202     }
4203 
4204     synchronized (writestate) {
4205       if (replaySeqId < lastReplayedOpenRegionSeqId) {
4206         LOG.warn(getRegionInfo().getEncodedName() + " : "
4207             + "Skipping replaying compaction event :" + TextFormat.shortDebugString(compaction)
4208             + " because its sequence id " + replaySeqId + " is smaller than this regions "
4209             + "lastReplayedOpenRegionSeqId of " + lastReplayedOpenRegionSeqId);
4210         return;
4211       }
4212       if (replaySeqId < lastReplayedCompactionSeqId) {
4213         LOG.warn(getRegionInfo().getEncodedName() + " : "
4214             + "Skipping replaying compaction event :" + TextFormat.shortDebugString(compaction)
4215             + " because its sequence id " + replaySeqId + " is smaller than this regions "
4216             + "lastReplayedCompactionSeqId of " + lastReplayedCompactionSeqId);
4217         return;
4218       } else {
4219         lastReplayedCompactionSeqId = replaySeqId;
4220       }
4221 
4222       if (LOG.isDebugEnabled()) {
4223         LOG.debug(getRegionInfo().getEncodedName() + " : "
4224             + "Replaying compaction marker " + TextFormat.shortDebugString(compaction)
4225             + " with seqId=" + replaySeqId + " and lastReplayedOpenRegionSeqId="
4226             + lastReplayedOpenRegionSeqId);
4227       }
4228 
4229       startRegionOperation(Operation.REPLAY_EVENT);
4230       try {
4231         Store store = this.getStore(compaction.getFamilyName().toByteArray());
4232         if (store == null) {
4233           LOG.warn(getRegionInfo().getEncodedName() + " : "
4234               + "Found Compaction WAL edit for deleted family:"
4235               + Bytes.toString(compaction.getFamilyName().toByteArray()));
4236           return;
4237         }
4238         store.replayCompactionMarker(compaction, pickCompactionFiles, removeFiles);
4239         logRegionFiles();
4240       } catch (FileNotFoundException ex) {
4241         LOG.warn(getRegionInfo().getEncodedName() + " : "
4242             + "At least one of the store files in compaction: "
4243             + TextFormat.shortDebugString(compaction)
4244             + " doesn't exist any more. Skip loading the file(s)", ex);
4245       } finally {
4246         closeRegionOperation(Operation.REPLAY_EVENT);
4247       }
4248     }
4249   }
4250 
4251   void replayWALFlushMarker(FlushDescriptor flush, long replaySeqId) throws IOException {
4252     checkTargetRegion(flush.getEncodedRegionName().toByteArray(),
4253       "Flush marker from WAL ", flush);
4254 
4255     if (ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {
4256       return; // if primary nothing to do
4257     }
4258 
4259     if (LOG.isDebugEnabled()) {
4260       LOG.debug(getRegionInfo().getEncodedName() + " : "
4261           + "Replaying flush marker " + TextFormat.shortDebugString(flush));
4262     }
4263 
4264     startRegionOperation(Operation.REPLAY_EVENT); // use region close lock to guard against close
4265     try {
4266       FlushAction action = flush.getAction();
4267       switch (action) {
4268       case START_FLUSH:
4269         replayWALFlushStartMarker(flush);
4270         break;
4271       case COMMIT_FLUSH:
4272         replayWALFlushCommitMarker(flush);
4273         break;
4274       case ABORT_FLUSH:
4275         replayWALFlushAbortMarker(flush);
4276         break;
4277       case CANNOT_FLUSH:
4278         replayWALFlushCannotFlushMarker(flush, replaySeqId);
4279         break;
4280       default:
4281         LOG.warn(getRegionInfo().getEncodedName() + " : " +
4282           "Received a flush event with unknown action, ignoring. " +
4283           TextFormat.shortDebugString(flush));
4284         break;
4285       }
4286 
4287       logRegionFiles();
4288     } finally {
4289       closeRegionOperation(Operation.REPLAY_EVENT);
4290     }
4291   }
4292 
4293   /** Replay the flush marker from primary region by creating a corresponding snapshot of
4294    * the store memstores, only if the memstores do not have a higher seqId from an earlier wal
4295    * edit (because the events may be coming out of order).
4296    */
4297   @VisibleForTesting
4298   PrepareFlushResult replayWALFlushStartMarker(FlushDescriptor flush) throws IOException {
4299     long flushSeqId = flush.getFlushSequenceNumber();
4300 
4301     HashSet<Store> storesToFlush = new HashSet<Store>();
4302     for (StoreFlushDescriptor storeFlush : flush.getStoreFlushesList()) {
4303       byte[] family = storeFlush.getFamilyName().toByteArray();
4304       Store store = getStore(family);
4305       if (store == null) {
4306         LOG.warn(getRegionInfo().getEncodedName() + " : "
4307           + "Received a flush start marker from primary, but the family is not found. Ignoring"
4308           + " StoreFlushDescriptor:" + TextFormat.shortDebugString(storeFlush));
4309         continue;
4310       }
4311       storesToFlush.add(store);
4312     }
4313 
4314     MonitoredTask status = TaskMonitor.get().createStatus("Preparing flush " + this);
4315 
4316     // we will use writestate as a coarse-grain lock for all the replay events
4317     // (flush, compaction, region open etc)
4318     synchronized (writestate) {
4319       try {
4320         if (flush.getFlushSequenceNumber() < lastReplayedOpenRegionSeqId) {
4321           LOG.warn(getRegionInfo().getEncodedName() + " : "
4322               + "Skipping replaying flush event :" + TextFormat.shortDebugString(flush)
4323               + " because its sequence id is smaller than this regions lastReplayedOpenRegionSeqId "
4324               + " of " + lastReplayedOpenRegionSeqId);
4325           return null;
4326         }
4327         if (numMutationsWithoutWAL.get() > 0) {
4328           numMutationsWithoutWAL.set(0);
4329           dataInMemoryWithoutWAL.set(0);
4330         }
4331 
4332         if (!writestate.flushing) {
4333           // we do not have an active snapshot and corresponding this.prepareResult. This means
4334           // we can just snapshot our memstores and continue as normal.
4335 
4336           // invoke prepareFlushCache. Send null as wal since we do not want the flush events in wal
4337           PrepareFlushResult prepareResult = internalPrepareFlushCache(null,
4338             flushSeqId, storesToFlush, status, false);
4339           if (prepareResult.result == null) {
4340             // save the PrepareFlushResult so that we can use it later from commit flush
4341             this.writestate.flushing = true;
4342             this.prepareFlushResult = prepareResult;
4343             status.markComplete("Flush prepare successful");
4344             if (LOG.isDebugEnabled()) {
4345               LOG.debug(getRegionInfo().getEncodedName() + " : "
4346                   + " Prepared flush with seqId:" + flush.getFlushSequenceNumber());
4347             }
4348           } else {
4349             // special case empty memstore. We will still save the flush result in this case, since
4350             // our memstore ie empty, but the primary is still flushing
4351             if (prepareResult.getResult().getResult() ==
4352                   FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY) {
4353               this.writestate.flushing = true;
4354               this.prepareFlushResult = prepareResult;
4355               if (LOG.isDebugEnabled()) {
4356                 LOG.debug(getRegionInfo().getEncodedName() + " : "
4357                   + " Prepared empty flush with seqId:" + flush.getFlushSequenceNumber());
4358               }
4359             }
4360             status.abort("Flush prepare failed with " + prepareResult.result);
4361             // nothing much to do. prepare flush failed because of some reason.
4362           }
4363           return prepareResult;
4364         } else {
4365           // we already have an active snapshot.
4366           if (flush.getFlushSequenceNumber() == this.prepareFlushResult.flushOpSeqId) {
4367             // They define the same flush. Log and continue.
4368             LOG.warn(getRegionInfo().getEncodedName() + " : "
4369                 + "Received a flush prepare marker with the same seqId: " +
4370                 + flush.getFlushSequenceNumber() + " before clearing the previous one with seqId: "
4371                 + prepareFlushResult.flushOpSeqId + ". Ignoring");
4372             // ignore
4373           } else if (flush.getFlushSequenceNumber() < this.prepareFlushResult.flushOpSeqId) {
4374             // We received a flush with a smaller seqNum than what we have prepared. We can only
4375             // ignore this prepare flush request.
4376             LOG.warn(getRegionInfo().getEncodedName() + " : "
4377                 + "Received a flush prepare marker with a smaller seqId: " +
4378                 + flush.getFlushSequenceNumber() + " before clearing the previous one with seqId: "
4379                 + prepareFlushResult.flushOpSeqId + ". Ignoring");
4380             // ignore
4381           } else {
4382             // We received a flush with a larger seqNum than what we have prepared
4383             LOG.warn(getRegionInfo().getEncodedName() + " : "
4384                 + "Received a flush prepare marker with a larger seqId: " +
4385                 + flush.getFlushSequenceNumber() + " before clearing the previous one with seqId: "
4386                 + prepareFlushResult.flushOpSeqId + ". Ignoring");
4387             // We do not have multiple active snapshots in the memstore or a way to merge current
4388             // memstore snapshot with the contents and resnapshot for now. We cannot take
4389             // another snapshot and drop the previous one because that will cause temporary
4390             // data loss in the secondary. So we ignore this for now, deferring the resolution
4391             // to happen when we see the corresponding flush commit marker. If we have a memstore
4392             // snapshot with x, and later received another prepare snapshot with y (where x < y),
4393             // when we see flush commit for y, we will drop snapshot for x, and can also drop all
4394             // the memstore edits if everything in memstore is < y. This is the usual case for
4395             // RS crash + recovery where we might see consequtive prepare flush wal markers.
4396             // Otherwise, this will cause more memory to be used in secondary replica until a
4397             // further prapare + commit flush is seen and replayed.
4398           }
4399         }
4400       } finally {
4401         status.cleanup();
4402         writestate.notifyAll();
4403       }
4404     }
4405     return null;
4406   }
4407 
4408   @VisibleForTesting
4409   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NN_NAKED_NOTIFY",
4410     justification="Intentional; post memstore flush")
4411   void replayWALFlushCommitMarker(FlushDescriptor flush) throws IOException {
4412     MonitoredTask status = TaskMonitor.get().createStatus("Committing flush " + this);
4413 
4414     // check whether we have the memstore snapshot with the corresponding seqId. Replay to
4415     // secondary region replicas are in order, except for when the region moves or then the
4416     // region server crashes. In those cases, we may receive replay requests out of order from
4417     // the original seqIds.
4418     synchronized (writestate) {
4419       try {
4420         if (flush.getFlushSequenceNumber() < lastReplayedOpenRegionSeqId) {
4421           LOG.warn(getRegionInfo().getEncodedName() + " : "
4422             + "Skipping replaying flush event :" + TextFormat.shortDebugString(flush)
4423             + " because its sequence id is smaller than this regions lastReplayedOpenRegionSeqId "
4424             + " of " + lastReplayedOpenRegionSeqId);
4425           return;
4426         }
4427 
4428         if (writestate.flushing) {
4429           PrepareFlushResult prepareFlushResult = this.prepareFlushResult;
4430           if (flush.getFlushSequenceNumber() == prepareFlushResult.flushOpSeqId) {
4431             if (LOG.isDebugEnabled()) {
4432               LOG.debug(getRegionInfo().getEncodedName() + " : "
4433                   + "Received a flush commit marker with seqId:" + flush.getFlushSequenceNumber()
4434                   + " and a previous prepared snapshot was found");
4435             }
4436             // This is the regular case where we received commit flush after prepare flush
4437             // corresponding to the same seqId.
4438             replayFlushInStores(flush, prepareFlushResult, true);
4439 
4440             // Set down the memstore size by amount of flush.
4441             this.addAndGetGlobalMemstoreSize(-prepareFlushResult.totalFlushableSize);
4442 
4443             this.prepareFlushResult = null;
4444             writestate.flushing = false;
4445           } else if (flush.getFlushSequenceNumber() < prepareFlushResult.flushOpSeqId) {
4446             // This should not happen normally. However, lets be safe and guard against these cases
4447             // we received a flush commit with a smaller seqId than what we have prepared
4448             // we will pick the flush file up from this commit (if we have not seen it), but we
4449             // will not drop the memstore
4450             LOG.warn(getRegionInfo().getEncodedName() + " : "
4451                 + "Received a flush commit marker with smaller seqId: "
4452                 + flush.getFlushSequenceNumber() + " than what we have prepared with seqId: "
4453                 + prepareFlushResult.flushOpSeqId + ". Picking up new file, but not dropping"
4454                 +"  prepared memstore snapshot");
4455             replayFlushInStores(flush, prepareFlushResult, false);
4456 
4457             // snapshot is not dropped, so memstore sizes should not be decremented
4458             // we still have the prepared snapshot, flushing should still be true
4459           } else {
4460             // This should not happen normally. However, lets be safe and guard against these cases
4461             // we received a flush commit with a larger seqId than what we have prepared
4462             // we will pick the flush file for this. We will also obtain the updates lock and
4463             // look for contents of the memstore to see whether we have edits after this seqId.
4464             // If not, we will drop all the memstore edits and the snapshot as well.
4465             LOG.warn(getRegionInfo().getEncodedName() + " : "
4466                 + "Received a flush commit marker with larger seqId: "
4467                 + flush.getFlushSequenceNumber() + " than what we have prepared with seqId: " +
4468                 prepareFlushResult.flushOpSeqId + ". Picking up new file and dropping prepared"
4469                 +" memstore snapshot");
4470 
4471             replayFlushInStores(flush, prepareFlushResult, true);
4472 
4473             // Set down the memstore size by amount of flush.
4474             this.addAndGetGlobalMemstoreSize(-prepareFlushResult.totalFlushableSize);
4475 
4476             // Inspect the memstore contents to see whether the memstore contains only edits
4477             // with seqId smaller than the flush seqId. If so, we can discard those edits.
4478             dropMemstoreContentsForSeqId(flush.getFlushSequenceNumber(), null);
4479 
4480             this.prepareFlushResult = null;
4481             writestate.flushing = false;
4482           }
4483           // If we were waiting for observing a flush or region opening event for not showing
4484           // partial data after a secondary region crash, we can allow reads now. We can only make
4485           // sure that we are not showing partial data (for example skipping some previous edits)
4486           // until we observe a full flush start and flush commit. So if we were not able to find
4487           // a previous flush we will not enable reads now.
4488           this.setReadsEnabled(true);
4489         } else {
4490           LOG.warn(getRegionInfo().getEncodedName() + " : "
4491               + "Received a flush commit marker with seqId:" + flush.getFlushSequenceNumber()
4492               + ", but no previous prepared snapshot was found");
4493           // There is no corresponding prepare snapshot from before.
4494           // We will pick up the new flushed file
4495           replayFlushInStores(flush, null, false);
4496 
4497           // Inspect the memstore contents to see whether the memstore contains only edits
4498           // with seqId smaller than the flush seqId. If so, we can discard those edits.
4499           dropMemstoreContentsForSeqId(flush.getFlushSequenceNumber(), null);
4500         }
4501 
4502         status.markComplete("Flush commit successful");
4503 
4504         // Update the last flushed sequence id for region.
4505         this.maxFlushedSeqId = flush.getFlushSequenceNumber();
4506 
4507         // advance the mvcc read point so that the new flushed file is visible.
4508         mvcc.advanceTo(flush.getFlushSequenceNumber());
4509 
4510       } catch (FileNotFoundException ex) {
4511         LOG.warn(getRegionInfo().getEncodedName() + " : "
4512             + "At least one of the store files in flush: " + TextFormat.shortDebugString(flush)
4513             + " doesn't exist any more. Skip loading the file(s)", ex);
4514       }
4515       finally {
4516         status.cleanup();
4517         writestate.notifyAll();
4518       }
4519     }
4520 
4521     // C. Finally notify anyone waiting on memstore to clear:
4522     // e.g. checkResources().
4523     synchronized (this) {
4524       notifyAll(); // FindBugs NN_NAKED_NOTIFY
4525     }
4526   }
4527 
4528   /**
4529    * Replays the given flush descriptor by opening the flush files in stores and dropping the
4530    * memstore snapshots if requested.
4531    * @param flush
4532    * @param prepareFlushResult
4533    * @param dropMemstoreSnapshot
4534    * @throws IOException
4535    */
4536   private void replayFlushInStores(FlushDescriptor flush, PrepareFlushResult prepareFlushResult,
4537       boolean dropMemstoreSnapshot)
4538       throws IOException {
4539     for (StoreFlushDescriptor storeFlush : flush.getStoreFlushesList()) {
4540       byte[] family = storeFlush.getFamilyName().toByteArray();
4541       Store store = getStore(family);
4542       if (store == null) {
4543         LOG.warn(getRegionInfo().getEncodedName() + " : "
4544             + "Received a flush commit marker from primary, but the family is not found."
4545             + "Ignoring StoreFlushDescriptor:" + storeFlush);
4546         continue;
4547       }
4548       List<String> flushFiles = storeFlush.getFlushOutputList();
4549       StoreFlushContext ctx = null;
4550       long startTime = EnvironmentEdgeManager.currentTime();
4551       if (prepareFlushResult == null || prepareFlushResult.storeFlushCtxs == null) {
4552         ctx = store.createFlushContext(flush.getFlushSequenceNumber());
4553       } else {
4554         ctx = prepareFlushResult.storeFlushCtxs.get(family);
4555         startTime = prepareFlushResult.startTime;
4556       }
4557 
4558       if (ctx == null) {
4559         LOG.warn(getRegionInfo().getEncodedName() + " : "
4560             + "Unexpected: flush commit marker received from store "
4561             + Bytes.toString(family) + " but no associated flush context. Ignoring");
4562         continue;
4563       }
4564 
4565       ctx.replayFlush(flushFiles, dropMemstoreSnapshot); // replay the flush
4566 
4567       // Record latest flush time
4568       this.lastStoreFlushTimeMap.put(store, startTime);
4569     }
4570   }
4571 
4572   /**
4573    * Drops the memstore contents after replaying a flush descriptor or region open event replay
4574    * if the memstore edits have seqNums smaller than the given seq id
4575    * @throws IOException
4576    */
4577   private long dropMemstoreContentsForSeqId(long seqId, Store store) throws IOException {
4578     long totalFreedSize = 0;
4579     this.updatesLock.writeLock().lock();
4580     try {
4581 
4582       long currentSeqId = mvcc.getReadPoint();
4583       if (seqId >= currentSeqId) {
4584         // then we can drop the memstore contents since everything is below this seqId
4585         LOG.info(getRegionInfo().getEncodedName() + " : "
4586             + "Dropping memstore contents as well since replayed flush seqId: "
4587             + seqId + " is greater than current seqId:" + currentSeqId);
4588 
4589         // Prepare flush (take a snapshot) and then abort (drop the snapshot)
4590         if (store == null) {
4591           for (Store s : stores.values()) {
4592             totalFreedSize += doDropStoreMemstoreContentsForSeqId(s, currentSeqId);
4593           }
4594         } else {
4595           totalFreedSize += doDropStoreMemstoreContentsForSeqId(store, currentSeqId);
4596         }
4597       } else {
4598         LOG.info(getRegionInfo().getEncodedName() + " : "
4599             + "Not dropping memstore contents since replayed flush seqId: "
4600             + seqId + " is smaller than current seqId:" + currentSeqId);
4601       }
4602     } finally {
4603       this.updatesLock.writeLock().unlock();
4604     }
4605     return totalFreedSize;
4606   }
4607 
4608   private long doDropStoreMemstoreContentsForSeqId(Store s, long currentSeqId) throws IOException {
4609     long snapshotSize = s.getFlushableSize();
4610     this.addAndGetGlobalMemstoreSize(-snapshotSize);
4611     StoreFlushContext ctx = s.createFlushContext(currentSeqId);
4612     ctx.prepare();
4613     ctx.abort();
4614     return snapshotSize;
4615   }
4616 
4617   private void replayWALFlushAbortMarker(FlushDescriptor flush) {
4618     // nothing to do for now. A flush abort will cause a RS abort which means that the region
4619     // will be opened somewhere else later. We will see the region open event soon, and replaying
4620     // that will drop the snapshot
4621   }
4622 
4623   private void replayWALFlushCannotFlushMarker(FlushDescriptor flush, long replaySeqId) {
4624     synchronized (writestate) {
4625       if (this.lastReplayedOpenRegionSeqId > replaySeqId) {
4626         LOG.warn(getRegionInfo().getEncodedName() + " : "
4627           + "Skipping replaying flush event :" + TextFormat.shortDebugString(flush)
4628           + " because its sequence id " + replaySeqId + " is smaller than this regions "
4629           + "lastReplayedOpenRegionSeqId of " + lastReplayedOpenRegionSeqId);
4630         return;
4631       }
4632 
4633       // If we were waiting for observing a flush or region opening event for not showing partial
4634       // data after a secondary region crash, we can allow reads now. This event means that the
4635       // primary was not able to flush because memstore is empty when we requested flush. By the
4636       // time we observe this, we are guaranteed to have up to date seqId with our previous
4637       // assignment.
4638       this.setReadsEnabled(true);
4639     }
4640   }
4641 
4642   @VisibleForTesting
4643   PrepareFlushResult getPrepareFlushResult() {
4644     return prepareFlushResult;
4645   }
4646 
4647   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NN_NAKED_NOTIFY",
4648       justification="Intentional; cleared the memstore")
4649   void replayWALRegionEventMarker(RegionEventDescriptor regionEvent) throws IOException {
4650     checkTargetRegion(regionEvent.getEncodedRegionName().toByteArray(),
4651       "RegionEvent marker from WAL ", regionEvent);
4652 
4653     startRegionOperation(Operation.REPLAY_EVENT);
4654     try {
4655       if (ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {
4656         return; // if primary nothing to do
4657       }
4658 
4659       if (regionEvent.getEventType() == EventType.REGION_CLOSE) {
4660         // nothing to do on REGION_CLOSE for now.
4661         return;
4662       }
4663       if (regionEvent.getEventType() != EventType.REGION_OPEN) {
4664         LOG.warn(getRegionInfo().getEncodedName() + " : "
4665             + "Unknown region event received, ignoring :"
4666             + TextFormat.shortDebugString(regionEvent));
4667         return;
4668       }
4669 
4670       if (LOG.isDebugEnabled()) {
4671         LOG.debug(getRegionInfo().getEncodedName() + " : "
4672           + "Replaying region open event marker " + TextFormat.shortDebugString(regionEvent));
4673       }
4674 
4675       // we will use writestate as a coarse-grain lock for all the replay events
4676       synchronized (writestate) {
4677         // Replication can deliver events out of order when primary region moves or the region
4678         // server crashes, since there is no coordination between replication of different wal files
4679         // belonging to different region servers. We have to safe guard against this case by using
4680         // region open event's seqid. Since this is the first event that the region puts (after
4681         // possibly flushing recovered.edits), after seeing this event, we can ignore every edit
4682         // smaller than this seqId
4683         if (this.lastReplayedOpenRegionSeqId <= regionEvent.getLogSequenceNumber()) {
4684           this.lastReplayedOpenRegionSeqId = regionEvent.getLogSequenceNumber();
4685         } else {
4686           LOG.warn(getRegionInfo().getEncodedName() + " : "
4687             + "Skipping replaying region event :" + TextFormat.shortDebugString(regionEvent)
4688             + " because its sequence id is smaller than this regions lastReplayedOpenRegionSeqId "
4689             + " of " + lastReplayedOpenRegionSeqId);
4690           return;
4691         }
4692 
4693         // region open lists all the files that the region has at the time of the opening. Just pick
4694         // all the files and drop prepared flushes and empty memstores
4695         for (StoreDescriptor storeDescriptor : regionEvent.getStoresList()) {
4696           // stores of primary may be different now
4697           byte[] family = storeDescriptor.getFamilyName().toByteArray();
4698           Store store = getStore(family);
4699           if (store == null) {
4700             LOG.warn(getRegionInfo().getEncodedName() + " : "
4701                 + "Received a region open marker from primary, but the family is not found. "
4702                 + "Ignoring. StoreDescriptor:" + storeDescriptor);
4703             continue;
4704           }
4705 
4706           long storeSeqId = store.getMaxSequenceId();
4707           List<String> storeFiles = storeDescriptor.getStoreFileList();
4708           try {
4709             store.refreshStoreFiles(storeFiles); // replace the files with the new ones
4710           } catch (FileNotFoundException ex) {
4711             LOG.warn(getRegionInfo().getEncodedName() + " : "
4712                     + "At least one of the store files: " + storeFiles
4713                     + " doesn't exist any more. Skip loading the file(s)", ex);
4714             continue;
4715           }
4716           if (store.getMaxSequenceId() != storeSeqId) {
4717             // Record latest flush time if we picked up new files
4718             lastStoreFlushTimeMap.put(store, EnvironmentEdgeManager.currentTime());
4719           }
4720 
4721           if (writestate.flushing) {
4722             // only drop memstore snapshots if they are smaller than last flush for the store
4723             if (this.prepareFlushResult.flushOpSeqId <= regionEvent.getLogSequenceNumber()) {
4724               StoreFlushContext ctx = this.prepareFlushResult.storeFlushCtxs == null ?
4725                   null : this.prepareFlushResult.storeFlushCtxs.get(family);
4726               if (ctx != null) {
4727                 long snapshotSize = store.getFlushableSize();
4728                 ctx.abort();
4729                 this.addAndGetGlobalMemstoreSize(-snapshotSize);
4730                 this.prepareFlushResult.storeFlushCtxs.remove(family);
4731               }
4732             }
4733           }
4734 
4735           // Drop the memstore contents if they are now smaller than the latest seen flushed file
4736           dropMemstoreContentsForSeqId(regionEvent.getLogSequenceNumber(), store);
4737           if (storeSeqId > this.maxFlushedSeqId) {
4738             this.maxFlushedSeqId = storeSeqId;
4739           }
4740         }
4741 
4742         // if all stores ended up dropping their snapshots, we can safely drop the
4743         // prepareFlushResult
4744         dropPrepareFlushIfPossible();
4745 
4746         // advance the mvcc read point so that the new flushed file is visible.
4747         mvcc.await();
4748 
4749         // If we were waiting for observing a flush or region opening event for not showing partial
4750         // data after a secondary region crash, we can allow reads now.
4751         this.setReadsEnabled(true);
4752 
4753         // C. Finally notify anyone waiting on memstore to clear:
4754         // e.g. checkResources().
4755         synchronized (this) {
4756           notifyAll(); // FindBugs NN_NAKED_NOTIFY
4757         }
4758       }
4759       logRegionFiles();
4760     } finally {
4761       closeRegionOperation(Operation.REPLAY_EVENT);
4762     }
4763   }
4764 
4765   void replayWALBulkLoadEventMarker(WALProtos.BulkLoadDescriptor bulkLoadEvent) throws IOException {
4766     checkTargetRegion(bulkLoadEvent.getEncodedRegionName().toByteArray(),
4767       "BulkLoad marker from WAL ", bulkLoadEvent);
4768 
4769     if (ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {
4770       return; // if primary nothing to do
4771     }
4772 
4773     if (LOG.isDebugEnabled()) {
4774       LOG.debug(getRegionInfo().getEncodedName() + " : "
4775               +  "Replaying bulkload event marker " + TextFormat.shortDebugString(bulkLoadEvent));
4776     }
4777     // check if multiple families involved
4778     boolean multipleFamilies = false;
4779     byte[] family = null;
4780     for (StoreDescriptor storeDescriptor : bulkLoadEvent.getStoresList()) {
4781       byte[] fam = storeDescriptor.getFamilyName().toByteArray();
4782       if (family == null) {
4783         family = fam;
4784       } else if (!Bytes.equals(family, fam)) {
4785         multipleFamilies = true;
4786         break;
4787       }
4788     }
4789 
4790     startBulkRegionOperation(multipleFamilies);
4791     try {
4792       // we will use writestate as a coarse-grain lock for all the replay events
4793       synchronized (writestate) {
4794         // Replication can deliver events out of order when primary region moves or the region
4795         // server crashes, since there is no coordination between replication of different wal files
4796         // belonging to different region servers. We have to safe guard against this case by using
4797         // region open event's seqid. Since this is the first event that the region puts (after
4798         // possibly flushing recovered.edits), after seeing this event, we can ignore every edit
4799         // smaller than this seqId
4800         if (bulkLoadEvent.getBulkloadSeqNum() >= 0
4801             && this.lastReplayedOpenRegionSeqId >= bulkLoadEvent.getBulkloadSeqNum()) {
4802           LOG.warn(getRegionInfo().getEncodedName() + " : "
4803               + "Skipping replaying bulkload event :"
4804               + TextFormat.shortDebugString(bulkLoadEvent)
4805               + " because its sequence id is smaller than this region's lastReplayedOpenRegionSeqId"
4806               + " =" + lastReplayedOpenRegionSeqId);
4807 
4808           return;
4809         }
4810 
4811         for (StoreDescriptor storeDescriptor : bulkLoadEvent.getStoresList()) {
4812           // stores of primary may be different now
4813           family = storeDescriptor.getFamilyName().toByteArray();
4814           Store store = getStore(family);
4815           if (store == null) {
4816             LOG.warn(getRegionInfo().getEncodedName() + " : "
4817                     + "Received a bulk load marker from primary, but the family is not found. "
4818                     + "Ignoring. StoreDescriptor:" + storeDescriptor);
4819             continue;
4820           }
4821 
4822           List<String> storeFiles = storeDescriptor.getStoreFileList();
4823           for (String storeFile : storeFiles) {
4824             StoreFileInfo storeFileInfo = null;
4825             try {
4826               storeFileInfo = fs.getStoreFileInfo(Bytes.toString(family), storeFile);
4827               store.bulkLoadHFile(storeFileInfo);
4828             } catch(FileNotFoundException ex) {
4829               LOG.warn(getRegionInfo().getEncodedName() + " : "
4830                       + ((storeFileInfo != null) ? storeFileInfo.toString() :
4831                             (new Path(Bytes.toString(family), storeFile)).toString())
4832                       + " doesn't exist any more. Skip loading the file");
4833             }
4834           }
4835         }
4836       }
4837       if (bulkLoadEvent.getBulkloadSeqNum() > 0) {
4838         mvcc.advanceTo(bulkLoadEvent.getBulkloadSeqNum());
4839       }
4840     } finally {
4841       closeBulkRegionOperation();
4842     }
4843   }
4844 
4845   /**
4846    * If all stores ended up dropping their snapshots, we can safely drop the prepareFlushResult
4847    */
4848   private void dropPrepareFlushIfPossible() {
4849     if (writestate.flushing) {
4850       boolean canDrop = true;
4851       if (prepareFlushResult.storeFlushCtxs != null) {
4852         for (Entry<byte[], StoreFlushContext> entry
4853             : prepareFlushResult.storeFlushCtxs.entrySet()) {
4854           Store store = getStore(entry.getKey());
4855           if (store == null) {
4856             continue;
4857           }
4858           if (store.getSnapshotSize() > 0) {
4859             canDrop = false;
4860             break;
4861           }
4862         }
4863       }
4864 
4865       // this means that all the stores in the region has finished flushing, but the WAL marker
4866       // may not have been written or we did not receive it yet.
4867       if (canDrop) {
4868         writestate.flushing = false;
4869         this.prepareFlushResult = null;
4870       }
4871     }
4872   }
4873 
4874   @Override
4875   public boolean refreshStoreFiles() throws IOException {
4876     return refreshStoreFiles(false);
4877   }
4878 
4879   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NN_NAKED_NOTIFY",
4880       justification="Notify is about post replay. Intentional")
4881   protected boolean refreshStoreFiles(boolean force) throws IOException {
4882     if (!force && ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {
4883       return false; // if primary nothing to do
4884     }
4885 
4886     if (LOG.isDebugEnabled()) {
4887       LOG.debug(getRegionInfo().getEncodedName() + " : "
4888           + "Refreshing store files to see whether we can free up memstore");
4889     }
4890 
4891     long totalFreedSize = 0;
4892 
4893     long smallestSeqIdInStores = Long.MAX_VALUE;
4894 
4895     startRegionOperation(); // obtain region close lock
4896     try {
4897       synchronized (writestate) {
4898         for (Store store : getStores()) {
4899           // TODO: some stores might see new data from flush, while others do not which
4900           // MIGHT break atomic edits across column families.
4901           long maxSeqIdBefore = store.getMaxSequenceId();
4902 
4903           // refresh the store files. This is similar to observing a region open wal marker.
4904           store.refreshStoreFiles();
4905 
4906           long storeSeqId = store.getMaxSequenceId();
4907           if (storeSeqId < smallestSeqIdInStores) {
4908             smallestSeqIdInStores = storeSeqId;
4909           }
4910 
4911           // see whether we can drop the memstore or the snapshot
4912           if (storeSeqId > maxSeqIdBefore) {
4913 
4914             if (writestate.flushing) {
4915               // only drop memstore snapshots if they are smaller than last flush for the store
4916               if (this.prepareFlushResult.flushOpSeqId <= storeSeqId) {
4917                 StoreFlushContext ctx = this.prepareFlushResult.storeFlushCtxs == null ?
4918                     null : this.prepareFlushResult.storeFlushCtxs.get(store.getFamily().getName());
4919                 if (ctx != null) {
4920                   long snapshotSize = store.getFlushableSize();
4921                   ctx.abort();
4922                   this.addAndGetGlobalMemstoreSize(-snapshotSize);
4923                   this.prepareFlushResult.storeFlushCtxs.remove(store.getFamily().getName());
4924                   totalFreedSize += snapshotSize;
4925                 }
4926               }
4927             }
4928 
4929             // Drop the memstore contents if they are now smaller than the latest seen flushed file
4930             totalFreedSize += dropMemstoreContentsForSeqId(storeSeqId, store);
4931           }
4932         }
4933 
4934         // if all stores ended up dropping their snapshots, we can safely drop the
4935         // prepareFlushResult
4936         dropPrepareFlushIfPossible();
4937 
4938         // advance the mvcc read point so that the new flushed files are visible.
4939           // either greater than flush seq number or they were already picked up via flush.
4940           for (Store s : getStores()) {
4941             mvcc.advanceTo(s.getMaxMemstoreTS());
4942           }
4943 
4944 
4945         // smallestSeqIdInStores is the seqId that we have a corresponding hfile for. We can safely
4946         // skip all edits that are to be replayed in the future with that has a smaller seqId
4947         // than this. We are updating lastReplayedOpenRegionSeqId so that we can skip all edits
4948         // that we have picked the flush files for
4949         if (this.lastReplayedOpenRegionSeqId < smallestSeqIdInStores) {
4950           this.lastReplayedOpenRegionSeqId = smallestSeqIdInStores;
4951         }
4952       }
4953       // C. Finally notify anyone waiting on memstore to clear:
4954       // e.g. checkResources().
4955       synchronized (this) {
4956         notifyAll(); // FindBugs NN_NAKED_NOTIFY
4957       }
4958       return totalFreedSize > 0;
4959     } finally {
4960       closeRegionOperation();
4961     }
4962   }
4963 
4964   private void logRegionFiles() {
4965     if (LOG.isTraceEnabled()) {
4966       LOG.trace(getRegionInfo().getEncodedName() + " : Store files for region: ");
4967       for (Store s : stores.values()) {
4968         Collection<StoreFile> storeFiles = s.getStorefiles();
4969         if (storeFiles == null) continue;
4970         for (StoreFile sf : storeFiles) {
4971           LOG.trace(getRegionInfo().getEncodedName() + " : " + sf);
4972         }
4973       }
4974     }
4975   }
4976 
4977   /** Checks whether the given regionName is either equal to our region, or that
4978    * the regionName is the primary region to our corresponding range for the secondary replica.
4979    */
4980   private void checkTargetRegion(byte[] encodedRegionName, String exceptionMsg, Object payload)
4981       throws WrongRegionException {
4982     if (Bytes.equals(this.getRegionInfo().getEncodedNameAsBytes(), encodedRegionName)) {
4983       return;
4984     }
4985 
4986     if (!RegionReplicaUtil.isDefaultReplica(this.getRegionInfo()) &&
4987         Bytes.equals(encodedRegionName,
4988           this.fs.getRegionInfoForFS().getEncodedNameAsBytes())) {
4989       return;
4990     }
4991 
4992     throw new WrongRegionException(exceptionMsg + payload
4993       + " targetted for region " + Bytes.toStringBinary(encodedRegionName)
4994       + " does not match this region: " + this.getRegionInfo());
4995   }
4996 
4997   /**
4998    * Used by tests
4999    * @param s Store to add edit too.
5000    * @param cell Cell to add.
5001    * @return True if we should flush.
5002    */
5003   protected boolean restoreEdit(final Store s, final Cell cell) {
5004     long kvSize = s.add(cell);
5005     if (this.rsAccounting != null) {
5006       rsAccounting.addAndGetRegionReplayEditsSize(getRegionInfo().getRegionName(), kvSize);
5007     }
5008     return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize));
5009   }
5010 
5011   /*
5012    * @param fs
5013    * @param p File to check.
5014    * @return True if file was zero-length (and if so, we'll delete it in here).
5015    * @throws IOException
5016    */
5017   private static boolean isZeroLengthThenDelete(final FileSystem fs, final Path p)
5018       throws IOException {
5019     FileStatus stat = fs.getFileStatus(p);
5020     if (stat.getLen() > 0) return false;
5021     LOG.warn("File " + p + " is zero-length, deleting.");
5022     fs.delete(p, false);
5023     return true;
5024   }
5025 
5026   protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException {
5027     if (family.isMobEnabled()) {
5028       if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
5029         throw new IOException("A minimum HFile version of "
5030             + HFile.MIN_FORMAT_VERSION_WITH_TAGS
5031             + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY
5032             + " accordingly.");
5033       }
5034       return new HMobStore(this, family, this.conf);
5035     }
5036     return new HStore(this, family, this.conf);
5037   }
5038 
5039   @Override
5040   public Store getStore(final byte[] column) {
5041     return this.stores.get(column);
5042   }
5043 
5044   /**
5045    * Return HStore instance. Does not do any copy: as the number of store is limited, we
5046    *  iterate on the list.
5047    */
5048   private Store getStore(Cell cell) {
5049     for (Map.Entry<byte[], Store> famStore : stores.entrySet()) {
5050       if (Bytes.equals(
5051           cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
5052           famStore.getKey(), 0, famStore.getKey().length)) {
5053         return famStore.getValue();
5054       }
5055     }
5056 
5057     return null;
5058   }
5059 
5060   @Override
5061   public List<Store> getStores() {
5062     List<Store> list = new ArrayList<Store>(stores.size());
5063     list.addAll(stores.values());
5064     return list;
5065   }
5066 
5067   @Override
5068   public List<String> getStoreFileList(final byte [][] columns)
5069     throws IllegalArgumentException {
5070     List<String> storeFileNames = new ArrayList<String>();
5071     synchronized(closeLock) {
5072       for(byte[] column : columns) {
5073         Store store = this.stores.get(column);
5074         if (store == null) {
5075           throw new IllegalArgumentException("No column family : " +
5076               new String(column) + " available");
5077         }
5078         Collection<StoreFile> storeFiles = store.getStorefiles();
5079         if (storeFiles == null) continue;
5080         for (StoreFile storeFile: storeFiles) {
5081           storeFileNames.add(storeFile.getPath().toString());
5082         }
5083 
5084         logRegionFiles();
5085       }
5086     }
5087     return storeFileNames;
5088   }
5089 
5090   //////////////////////////////////////////////////////////////////////////////
5091   // Support code
5092   //////////////////////////////////////////////////////////////////////////////
5093 
5094   /** Make sure this is a valid row for the HRegion */
5095   void checkRow(final byte [] row, String op) throws IOException {
5096     if (!rowIsInRange(getRegionInfo(), row)) {
5097       throw new WrongRegionException("Requested row out of range for " +
5098           op + " on HRegion " + this + ", startKey='" +
5099           Bytes.toStringBinary(getRegionInfo().getStartKey()) + "', getEndKey()='" +
5100           Bytes.toStringBinary(getRegionInfo().getEndKey()) + "', row='" +
5101           Bytes.toStringBinary(row) + "'");
5102     }
5103   }
5104 
5105 
5106   /**
5107    * Get an exclusive ( write lock ) lock on a given row.
5108    * @param row Which row to lock.
5109    * @return A locked RowLock. The lock is exclusive and already aqquired.
5110    * @throws IOException
5111    */
5112   public RowLock getRowLock(byte[] row) throws IOException {
5113     return getRowLock(row, false);
5114   }
5115 
5116   /**
5117    *
5118    * Get a row lock for the specified row. All locks are reentrant.
5119    *
5120    * Before calling this function make sure that a region operation has already been
5121    * started (the calling thread has already acquired the region-close-guard lock).
5122    * @param row The row actions will be performed against
5123    * @param readLock is the lock reader or writer. True indicates that a non-exlcusive
5124    *                 lock is requested
5125    */
5126   public RowLock getRowLock(byte[] row, boolean readLock) throws IOException {
5127     // Make sure the row is inside of this region before getting the lock for it.
5128     checkRow(row, "row lock");
5129     // create an object to use a a key in the row lock map
5130     HashedBytes rowKey = new HashedBytes(row);
5131 
5132     RowLockContext rowLockContext = null;
5133     RowLockImpl result = null;
5134     TraceScope traceScope = null;
5135 
5136     // If we're tracing start a span to show how long this took.
5137     if (Trace.isTracing()) {
5138       traceScope = Trace.startSpan("HRegion.getRowLock");
5139       traceScope.getSpan().addTimelineAnnotation("Getting a " + (readLock?"readLock":"writeLock"));
5140     }
5141 
5142     try {
5143       // Keep trying until we have a lock or error out.
5144       // TODO: do we need to add a time component here?
5145       while (result == null) {
5146 
5147         // Try adding a RowLockContext to the lockedRows.
5148         // If we can add it then there's no other transactions currently running.
5149         rowLockContext = new RowLockContext(rowKey);
5150         RowLockContext existingContext = lockedRows.putIfAbsent(rowKey, rowLockContext);
5151 
5152         // if there was a running transaction then there's already a context.
5153         if (existingContext != null) {
5154           rowLockContext = existingContext;
5155         }
5156 
5157         // Now try an get the lock.
5158         //
5159         // This can fail as
5160         if (readLock) {
5161           result = rowLockContext.newReadLock();
5162         } else {
5163           result = rowLockContext.newWriteLock();
5164         }
5165       }
5166       if (!result.getLock().tryLock(this.rowLockWaitDuration, TimeUnit.MILLISECONDS)) {
5167         if (traceScope != null) {
5168           traceScope.getSpan().addTimelineAnnotation("Failed to get row lock");
5169         }
5170         result = null;
5171         // Clean up the counts just in case this was the thing keeping the context alive.
5172         rowLockContext.cleanUp();
5173         throw new IOException("Timed out waiting for lock for row: " + rowKey);
5174       }
5175       return result;
5176     } catch (InterruptedException ie) {
5177       LOG.warn("Thread interrupted waiting for lock on row: " + rowKey);
5178       InterruptedIOException iie = new InterruptedIOException();
5179       iie.initCause(ie);
5180       if (traceScope != null) {
5181         traceScope.getSpan().addTimelineAnnotation("Interrupted exception getting row lock");
5182       }
5183       Thread.currentThread().interrupt();
5184       throw iie;
5185     } finally {
5186       if (traceScope != null) {
5187         traceScope.close();
5188       }
5189     }
5190   }
5191 
5192   @Override
5193   public void releaseRowLocks(List<RowLock> rowLocks) {
5194     if (rowLocks != null) {
5195       for (RowLock rowLock : rowLocks) {
5196         rowLock.release();
5197       }
5198       rowLocks.clear();
5199     }
5200   }
5201 
5202   @VisibleForTesting
5203   class RowLockContext {
5204     private final HashedBytes row;
5205     final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(true);
5206     final AtomicBoolean usable = new AtomicBoolean(true);
5207     final AtomicInteger count = new AtomicInteger(0);
5208     final Object lock = new Object();
5209 
5210     RowLockContext(HashedBytes row) {
5211       this.row = row;
5212     }
5213 
5214     RowLockImpl newWriteLock() {
5215       Lock l = readWriteLock.writeLock();
5216       return getRowLock(l);
5217     }
5218     RowLockImpl newReadLock() {
5219       Lock l = readWriteLock.readLock();
5220       return getRowLock(l);
5221     }
5222 
5223     private RowLockImpl getRowLock(Lock l) {
5224       count.incrementAndGet();
5225       synchronized (lock) {
5226         if (usable.get()) {
5227           return new RowLockImpl(this, l);
5228         } else {
5229           return null;
5230         }
5231       }
5232     }
5233 
5234     void cleanUp() {
5235       long c = count.decrementAndGet();
5236       if (c <= 0) {
5237         synchronized (lock) {
5238           if (count.get() <= 0 ){
5239             usable.set(false);
5240             RowLockContext removed = lockedRows.remove(row);
5241             assert removed == this: "we should never remove a different context";
5242           }
5243         }
5244       }
5245     }
5246 
5247     @Override
5248     public String toString() {
5249       return "RowLockContext{" +
5250           "row=" + row +
5251           ", readWriteLock=" + readWriteLock +
5252           ", count=" + count +
5253           '}';
5254     }
5255   }
5256 
5257   /**
5258    * Class used to represent a lock on a row.
5259    */
5260   public static class RowLockImpl implements RowLock {
5261     private final RowLockContext context;
5262     private final Lock lock;
5263 
5264     public RowLockImpl(RowLockContext context, Lock lock) {
5265       this.context = context;
5266       this.lock = lock;
5267     }
5268 
5269     public Lock getLock() {
5270       return lock;
5271     }
5272 
5273     @VisibleForTesting
5274     public RowLockContext getContext() {
5275       return context;
5276     }
5277 
5278     @Override
5279     public void release() {
5280       lock.unlock();
5281       context.cleanUp();
5282     }
5283 
5284     @Override
5285     public String toString() {
5286       return "RowLockImpl{" +
5287           "context=" + context +
5288           ", lock=" + lock +
5289           '}';
5290     }
5291   }
5292 
5293   /**
5294    * Determines whether multiple column families are present
5295    * Precondition: familyPaths is not null
5296    *
5297    * @param familyPaths List of Pair<byte[] column family, String hfilePath>
5298    */
5299   private static boolean hasMultipleColumnFamilies(Collection<Pair<byte[], String>> familyPaths) {
5300     boolean multipleFamilies = false;
5301     byte[] family = null;
5302     for (Pair<byte[], String> pair : familyPaths) {
5303       byte[] fam = pair.getFirst();
5304       if (family == null) {
5305         family = fam;
5306       } else if (!Bytes.equals(family, fam)) {
5307         multipleFamilies = true;
5308         break;
5309       }
5310     }
5311     return multipleFamilies;
5312   }
5313 
5314   @Override
5315   public boolean bulkLoadHFiles(Collection<Pair<byte[], String>> familyPaths, boolean assignSeqId,
5316       BulkLoadListener bulkLoadListener) throws IOException {
5317     long seqId = -1;
5318     Map<byte[], List<Path>> storeFiles = new TreeMap<byte[], List<Path>>(Bytes.BYTES_COMPARATOR);
5319     Preconditions.checkNotNull(familyPaths);
5320     // we need writeLock for multi-family bulk load
5321     startBulkRegionOperation(hasMultipleColumnFamilies(familyPaths));
5322     try {
5323       this.writeRequestsCount.increment();
5324 
5325       // There possibly was a split that happened between when the split keys
5326       // were gathered and before the HRegion's write lock was taken.  We need
5327       // to validate the HFile region before attempting to bulk load all of them
5328       List<IOException> ioes = new ArrayList<IOException>();
5329       List<Pair<byte[], String>> failures = new ArrayList<Pair<byte[], String>>();
5330       for (Pair<byte[], String> p : familyPaths) {
5331         byte[] familyName = p.getFirst();
5332         String path = p.getSecond();
5333 
5334         Store store = getStore(familyName);
5335         if (store == null) {
5336           IOException ioe = new org.apache.hadoop.hbase.DoNotRetryIOException(
5337               "No such column family " + Bytes.toStringBinary(familyName));
5338           ioes.add(ioe);
5339         } else {
5340           try {
5341             store.assertBulkLoadHFileOk(new Path(path));
5342           } catch (WrongRegionException wre) {
5343             // recoverable (file doesn't fit in region)
5344             failures.add(p);
5345           } catch (IOException ioe) {
5346             // unrecoverable (hdfs problem)
5347             ioes.add(ioe);
5348           }
5349         }
5350       }
5351 
5352       // validation failed because of some sort of IO problem.
5353       if (ioes.size() != 0) {
5354         IOException e = MultipleIOException.createIOException(ioes);
5355         LOG.error("There were one or more IO errors when checking if the bulk load is ok.", e);
5356         throw e;
5357       }
5358 
5359       // validation failed, bail out before doing anything permanent.
5360       if (failures.size() != 0) {
5361         StringBuilder list = new StringBuilder();
5362         for (Pair<byte[], String> p : failures) {
5363           list.append("\n").append(Bytes.toString(p.getFirst())).append(" : ")
5364               .append(p.getSecond());
5365         }
5366         // problem when validating
5367         LOG.warn("There was a recoverable bulk load failure likely due to a" +
5368             " split.  These (family, HFile) pairs were not loaded: " + list);
5369         return false;
5370       }
5371 
5372       // We need to assign a sequential ID that's in between two memstores in order to preserve
5373       // the guarantee that all the edits lower than the highest sequential ID from all the
5374       // HFiles are flushed on disk. See HBASE-10958.  The sequence id returned when we flush is
5375       // guaranteed to be one beyond the file made when we flushed (or if nothing to flush, it is
5376       // a sequence id that we can be sure is beyond the last hfile written).
5377       if (assignSeqId) {
5378         FlushResult fs = flushcache(true, false);
5379         if (fs.isFlushSucceeded()) {
5380           seqId = ((FlushResultImpl)fs).flushSequenceId;
5381         } else if (fs.getResult() == FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY) {
5382           seqId = ((FlushResultImpl)fs).flushSequenceId;
5383         } else {
5384           throw new IOException("Could not bulk load with an assigned sequential ID because the "+
5385             "flush didn't run. Reason for not flushing: " + ((FlushResultImpl)fs).failureReason);
5386         }
5387       }
5388 
5389       for (Pair<byte[], String> p : familyPaths) {
5390         byte[] familyName = p.getFirst();
5391         String path = p.getSecond();
5392         Store store = getStore(familyName);
5393         try {
5394           String finalPath = path;
5395           if (bulkLoadListener != null) {
5396             finalPath = bulkLoadListener.prepareBulkLoad(familyName, path);
5397           }
5398           Path commitedStoreFile = store.bulkLoadHFile(finalPath, seqId);
5399 
5400           if(storeFiles.containsKey(familyName)) {
5401             storeFiles.get(familyName).add(commitedStoreFile);
5402           } else {
5403             List<Path> storeFileNames = new ArrayList<Path>();
5404             storeFileNames.add(commitedStoreFile);
5405             storeFiles.put(familyName, storeFileNames);
5406           }
5407           if (bulkLoadListener != null) {
5408             bulkLoadListener.doneBulkLoad(familyName, path);
5409           }
5410         } catch (IOException ioe) {
5411           // A failure here can cause an atomicity violation that we currently
5412           // cannot recover from since it is likely a failed HDFS operation.
5413 
5414           // TODO Need a better story for reverting partial failures due to HDFS.
5415           LOG.error("There was a partial failure due to IO when attempting to" +
5416               " load " + Bytes.toString(p.getFirst()) + " : " + p.getSecond(), ioe);
5417           if (bulkLoadListener != null) {
5418             try {
5419               bulkLoadListener.failedBulkLoad(familyName, path);
5420             } catch (Exception ex) {
5421               LOG.error("Error while calling failedBulkLoad for family " +
5422                   Bytes.toString(familyName) + " with path " + path, ex);
5423             }
5424           }
5425           throw ioe;
5426         }
5427       }
5428 
5429       return true;
5430     } finally {
5431       if (wal != null && !storeFiles.isEmpty()) {
5432         // write a bulk load event when not all hfiles are loaded
5433         try {
5434           WALProtos.BulkLoadDescriptor loadDescriptor = ProtobufUtil.toBulkLoadDescriptor(
5435               this.getRegionInfo().getTable(),
5436               ByteStringer.wrap(this.getRegionInfo().getEncodedNameAsBytes()), storeFiles, seqId);
5437           WALUtil.writeBulkLoadMarkerAndSync(wal, this.htableDescriptor, getRegionInfo(),
5438               loadDescriptor, mvcc);
5439         } catch (IOException ioe) {
5440           if (this.rsServices != null) {
5441             // Have to abort region server because some hfiles has been loaded but we can't write
5442             // the event into WAL
5443             this.rsServices.abort("Failed to write bulk load event into WAL.", ioe);
5444           }
5445         }
5446       }
5447 
5448       closeBulkRegionOperation();
5449     }
5450   }
5451 
5452   @Override
5453   public boolean equals(Object o) {
5454     return o instanceof HRegion && Bytes.equals(getRegionInfo().getRegionName(),
5455                                                 ((HRegion) o).getRegionInfo().getRegionName());
5456   }
5457 
5458   @Override
5459   public int hashCode() {
5460     return Bytes.hashCode(getRegionInfo().getRegionName());
5461   }
5462 
5463   @Override
5464   public String toString() {
5465     return getRegionInfo().getRegionNameAsString();
5466   }
5467 
5468   /**
5469    * RegionScannerImpl is used to combine scanners from multiple Stores (aka column families).
5470    */
5471   class RegionScannerImpl implements RegionScanner {
5472     // Package local for testability
5473     KeyValueHeap storeHeap = null;
5474     /** Heap of key-values that are not essential for the provided filters and are thus read
5475      * on demand, if on-demand column family loading is enabled.*/
5476     KeyValueHeap joinedHeap = null;
5477     /**
5478      * If the joined heap data gathering is interrupted due to scan limits, this will
5479      * contain the row for which we are populating the values.*/
5480     protected Cell joinedContinuationRow = null;
5481     private boolean filterClosed = false;
5482 
5483     protected final int isScan;
5484     protected final byte[] stopRow;
5485     protected final HRegion region;
5486 
5487     private final long readPt;
5488     private final long maxResultSize;
5489     private final ScannerContext defaultScannerContext;
5490     private final FilterWrapper filter;
5491 
5492     @Override
5493     public HRegionInfo getRegionInfo() {
5494       return region.getRegionInfo();
5495     }
5496 
5497     RegionScannerImpl(Scan scan, List<KeyValueScanner> additionalScanners, HRegion region)
5498         throws IOException {
5499       this.region = region;
5500       this.maxResultSize = scan.getMaxResultSize();
5501       if (scan.hasFilter()) {
5502         this.filter = new FilterWrapper(scan.getFilter());
5503       } else {
5504         this.filter = null;
5505       }
5506 
5507       /**
5508        * By default, calls to next/nextRaw must enforce the batch limit. Thus, construct a default
5509        * scanner context that can be used to enforce the batch limit in the event that a
5510        * ScannerContext is not specified during an invocation of next/nextRaw
5511        */
5512       defaultScannerContext = ScannerContext.newBuilder().setBatchLimit(scan.getBatch()).build();
5513 
5514       if (Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW) && !scan.isGetScan()) {
5515         this.stopRow = null;
5516       } else {
5517         this.stopRow = scan.getStopRow();
5518       }
5519       // If we are doing a get, we want to be [startRow,endRow] normally
5520       // it is [startRow,endRow) and if startRow=endRow we get nothing.
5521       this.isScan = scan.isGetScan() ? -1 : 0;
5522 
5523       // synchronize on scannerReadPoints so that nobody calculates
5524       // getSmallestReadPoint, before scannerReadPoints is updated.
5525       IsolationLevel isolationLevel = scan.getIsolationLevel();
5526       synchronized(scannerReadPoints) {
5527         this.readPt = getReadpoint(isolationLevel);
5528         scannerReadPoints.put(this, this.readPt);
5529       }
5530 
5531       // Here we separate all scanners into two lists - scanner that provide data required
5532       // by the filter to operate (scanners list) and all others (joinedScanners list).
5533       List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
5534       List<KeyValueScanner> joinedScanners = new ArrayList<KeyValueScanner>();
5535       if (additionalScanners != null) {
5536         scanners.addAll(additionalScanners);
5537       }
5538 
5539       for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
5540         Store store = stores.get(entry.getKey());
5541         KeyValueScanner scanner;
5542         try {
5543           scanner = store.getScanner(scan, entry.getValue(), this.readPt);
5544         } catch (FileNotFoundException e) {
5545           throw handleFileNotFound(e);
5546         }
5547         if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand()
5548           || this.filter.isFamilyEssential(entry.getKey())) {
5549           scanners.add(scanner);
5550         } else {
5551           joinedScanners.add(scanner);
5552         }
5553       }
5554       initializeKVHeap(scanners, joinedScanners, region);
5555     }
5556 
5557     protected void initializeKVHeap(List<KeyValueScanner> scanners,
5558         List<KeyValueScanner> joinedScanners, HRegion region)
5559         throws IOException {
5560       this.storeHeap = new KeyValueHeap(scanners, region.comparator);
5561       if (!joinedScanners.isEmpty()) {
5562         this.joinedHeap = new KeyValueHeap(joinedScanners, region.comparator);
5563       }
5564     }
5565 
5566     @Override
5567     public long getMaxResultSize() {
5568       return maxResultSize;
5569     }
5570 
5571     @Override
5572     public long getMvccReadPoint() {
5573       return this.readPt;
5574     }
5575 
5576     @Override
5577     public int getBatch() {
5578       return this.defaultScannerContext.getBatchLimit();
5579     }
5580 
5581     /**
5582      * Reset both the filter and the old filter.
5583      *
5584      * @throws IOException in case a filter raises an I/O exception.
5585      */
5586     protected void resetFilters() throws IOException {
5587       if (filter != null) {
5588         filter.reset();
5589       }
5590     }
5591 
5592     @Override
5593     public boolean next(List<Cell> outResults)
5594         throws IOException {
5595       // apply the batching limit by default
5596       return next(outResults, defaultScannerContext);
5597     }
5598 
5599     @Override
5600     public synchronized boolean next(List<Cell> outResults, ScannerContext scannerContext)
5601     throws IOException {
5602       if (this.filterClosed) {
5603         throw new UnknownScannerException("Scanner was closed (timed out?) " +
5604             "after we renewed it. Could be caused by a very slow scanner " +
5605             "or a lengthy garbage collection");
5606       }
5607       startRegionOperation(Operation.SCAN);
5608       readRequestsCount.increment();
5609       try {
5610         return nextRaw(outResults, scannerContext);
5611       } finally {
5612         closeRegionOperation(Operation.SCAN);
5613       }
5614     }
5615 
5616     @Override
5617     public boolean nextRaw(List<Cell> outResults) throws IOException {
5618       // Use the RegionScanner's context by default
5619       return nextRaw(outResults, defaultScannerContext);
5620     }
5621 
5622     @Override
5623     public boolean nextRaw(List<Cell> outResults, ScannerContext scannerContext)
5624         throws IOException {
5625       if (storeHeap == null) {
5626         // scanner is closed
5627         throw new UnknownScannerException("Scanner was closed");
5628       }
5629       boolean moreValues;
5630       if (outResults.isEmpty()) {
5631         // Usually outResults is empty. This is true when next is called
5632         // to handle scan or get operation.
5633         moreValues = nextInternal(outResults, scannerContext);
5634       } else {
5635         List<Cell> tmpList = new ArrayList<Cell>();
5636         moreValues = nextInternal(tmpList, scannerContext);
5637         outResults.addAll(tmpList);
5638       }
5639 
5640       // If the size limit was reached it means a partial Result is being returned. Returning a
5641       // partial Result means that we should not reset the filters; filters should only be reset in
5642       // between rows
5643       if (!scannerContext.partialResultFormed()) resetFilters();
5644 
5645       if (isFilterDoneInternal()) {
5646         moreValues = false;
5647       }
5648       return moreValues;
5649     }
5650 
5651     /**
5652      * @return true if more cells exist after this batch, false if scanner is done
5653      */
5654     private boolean populateFromJoinedHeap(List<Cell> results, ScannerContext scannerContext)
5655             throws IOException {
5656       assert joinedContinuationRow != null;
5657       boolean moreValues =
5658           populateResult(results, this.joinedHeap, scannerContext,
5659           joinedContinuationRow.getRowArray(), joinedContinuationRow.getRowOffset(),
5660           joinedContinuationRow.getRowLength());
5661 
5662       if (!scannerContext.checkAnyLimitReached(LimitScope.BETWEEN_CELLS)) {
5663         // We are done with this row, reset the continuation.
5664         joinedContinuationRow = null;
5665       }
5666       // As the data is obtained from two independent heaps, we need to
5667       // ensure that result list is sorted, because Result relies on that.
5668       Collections.sort(results, comparator);
5669       return moreValues;
5670     }
5671 
5672     /**
5673      * Fetches records with currentRow into results list, until next row, batchLimit (if not -1) is
5674      * reached, or remainingResultSize (if not -1) is reaced
5675      * @param heap KeyValueHeap to fetch data from.It must be positioned on correct row before call.
5676      * @param scannerContext
5677      * @param currentRow Byte array with key we are fetching.
5678      * @param offset offset for currentRow
5679      * @param length length for currentRow
5680      * @return state of last call to {@link KeyValueHeap#next()}
5681      */
5682     private boolean populateResult(List<Cell> results, KeyValueHeap heap,
5683         ScannerContext scannerContext, byte[] currentRow, int offset, short length)
5684         throws IOException {
5685       Cell nextKv;
5686       boolean moreCellsInRow = false;
5687       boolean tmpKeepProgress = scannerContext.getKeepProgress();
5688       // Scanning between column families and thus the scope is between cells
5689       LimitScope limitScope = LimitScope.BETWEEN_CELLS;
5690       try {
5691         do {
5692           // We want to maintain any progress that is made towards the limits while scanning across
5693           // different column families. To do this, we toggle the keep progress flag on during calls
5694           // to the StoreScanner to ensure that any progress made thus far is not wiped away.
5695           scannerContext.setKeepProgress(true);
5696           heap.next(results, scannerContext);
5697           scannerContext.setKeepProgress(tmpKeepProgress);
5698 
5699           nextKv = heap.peek();
5700           moreCellsInRow = moreCellsInRow(nextKv, currentRow, offset, length);
5701           if (!moreCellsInRow) incrementCountOfRowsScannedMetric(scannerContext);
5702 
5703           if (scannerContext.checkBatchLimit(limitScope)) {
5704             return scannerContext.setScannerState(NextState.BATCH_LIMIT_REACHED).hasMoreValues();
5705           } else if (scannerContext.checkSizeLimit(limitScope)) {
5706             ScannerContext.NextState state =
5707                 moreCellsInRow? NextState.SIZE_LIMIT_REACHED_MID_ROW: NextState.SIZE_LIMIT_REACHED;
5708             return scannerContext.setScannerState(state).hasMoreValues();
5709           } else if (scannerContext.checkTimeLimit(limitScope)) {
5710             ScannerContext.NextState state =
5711                 moreCellsInRow? NextState.TIME_LIMIT_REACHED_MID_ROW: NextState.TIME_LIMIT_REACHED;
5712             return scannerContext.setScannerState(state).hasMoreValues();
5713           }
5714         } while (moreCellsInRow);
5715       } catch (FileNotFoundException e) {
5716         throw handleFileNotFound(e);
5717       }
5718       return nextKv != null;
5719     }
5720 
5721     /**
5722      * Based on the nextKv in the heap, and the current row, decide whether or not there are more
5723      * cells to be read in the heap. If the row of the nextKv in the heap matches the current row
5724      * then there are more cells to be read in the row.
5725      * @param nextKv
5726      * @param currentRow
5727      * @param offset
5728      * @param length
5729      * @return true When there are more cells in the row to be read
5730      */
5731     private boolean moreCellsInRow(final Cell nextKv, byte[] currentRow, int offset,
5732         short length) {
5733       return nextKv != null && CellUtil.matchingRow(nextKv, currentRow, offset, length);
5734     }
5735 
5736     /*
5737      * @return True if a filter rules the scanner is over, done.
5738      */
5739     @Override
5740     public synchronized boolean isFilterDone() throws IOException {
5741       return isFilterDoneInternal();
5742     }
5743 
5744     private boolean isFilterDoneInternal() throws IOException {
5745       return this.filter != null && this.filter.filterAllRemaining();
5746     }
5747 
5748     private boolean nextInternal(List<Cell> results, ScannerContext scannerContext)
5749         throws IOException {
5750       if (!results.isEmpty()) {
5751         throw new IllegalArgumentException("First parameter should be an empty list");
5752       }
5753       if (scannerContext == null) {
5754         throw new IllegalArgumentException("Scanner context cannot be null");
5755       }
5756       RpcCallContext rpcCall = RpcServer.getCurrentCall();
5757 
5758       // Save the initial progress from the Scanner context in these local variables. The progress
5759       // may need to be reset a few times if rows are being filtered out so we save the initial
5760       // progress.
5761       int initialBatchProgress = scannerContext.getBatchProgress();
5762       long initialSizeProgress = scannerContext.getSizeProgress();
5763       long initialTimeProgress = scannerContext.getTimeProgress();
5764 
5765       // The loop here is used only when at some point during the next we determine
5766       // that due to effects of filters or otherwise, we have an empty row in the result.
5767       // Then we loop and try again. Otherwise, we must get out on the first iteration via return,
5768       // "true" if there's more data to read, "false" if there isn't (storeHeap is at a stop row,
5769       // and joinedHeap has no more data to read for the last row (if set, joinedContinuationRow).
5770       while (true) {
5771         // Starting to scan a new row. Reset the scanner progress according to whether or not
5772         // progress should be kept.
5773         if (scannerContext.getKeepProgress()) {
5774           // Progress should be kept. Reset to initial values seen at start of method invocation.
5775           scannerContext.setProgress(initialBatchProgress, initialSizeProgress,
5776             initialTimeProgress);
5777         } else {
5778           scannerContext.clearProgress();
5779         }
5780 
5781         if (rpcCall != null) {
5782           // If a user specifies a too-restrictive or too-slow scanner, the
5783           // client might time out and disconnect while the server side
5784           // is still processing the request. We should abort aggressively
5785           // in that case.
5786           long afterTime = rpcCall.disconnectSince();
5787           if (afterTime >= 0) {
5788             throw new CallerDisconnectedException(
5789                 "Aborting on region " + getRegionInfo().getRegionNameAsString() + ", call " +
5790                     this + " after " + afterTime + " ms, since " +
5791                     "caller disconnected");
5792           }
5793         }
5794 
5795         // Let's see what we have in the storeHeap.
5796         Cell current = this.storeHeap.peek();
5797 
5798         byte[] currentRow = null;
5799         int offset = 0;
5800         short length = 0;
5801         if (current != null) {
5802           currentRow = current.getRowArray();
5803           offset = current.getRowOffset();
5804           length = current.getRowLength();
5805         }
5806 
5807         boolean stopRow = isStopRow(currentRow, offset, length);
5808         // When has filter row is true it means that the all the cells for a particular row must be
5809         // read before a filtering decision can be made. This means that filters where hasFilterRow
5810         // run the risk of encountering out of memory errors in the case that they are applied to a
5811         // table that has very large rows.
5812         boolean hasFilterRow = this.filter != null && this.filter.hasFilterRow();
5813 
5814         // If filter#hasFilterRow is true, partial results are not allowed since allowing them
5815         // would prevent the filters from being evaluated. Thus, if it is true, change the
5816         // scope of any limits that could potentially create partial results to
5817         // LimitScope.BETWEEN_ROWS so that those limits are not reached mid-row
5818         if (hasFilterRow) {
5819           if (LOG.isTraceEnabled()) {
5820             LOG.trace("filter#hasFilterRow is true which prevents partial results from being "
5821                 + " formed. Changing scope of limits that may create partials");
5822           }
5823           scannerContext.setSizeLimitScope(LimitScope.BETWEEN_ROWS);
5824           scannerContext.setTimeLimitScope(LimitScope.BETWEEN_ROWS);
5825         }
5826 
5827         // Check if we were getting data from the joinedHeap and hit the limit.
5828         // If not, then it's main path - getting results from storeHeap.
5829         if (joinedContinuationRow == null) {
5830           // First, check if we are at a stop row. If so, there are no more results.
5831           if (stopRow) {
5832             if (hasFilterRow) {
5833               filter.filterRowCells(results);
5834             }
5835             return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
5836           }
5837 
5838           // Check if rowkey filter wants to exclude this row. If so, loop to next.
5839           // Technically, if we hit limits before on this row, we don't need this call.
5840           if (filterRowKey(currentRow, offset, length)) {
5841             incrementCountOfRowsFilteredMetric(scannerContext);
5842             // Typically the count of rows scanned is incremented inside #populateResult. However,
5843             // here we are filtering a row based purely on its row key, preventing us from calling
5844             // #populateResult. Thus, perform the necessary increment here to rows scanned metric
5845             incrementCountOfRowsScannedMetric(scannerContext);
5846             boolean moreRows = nextRow(scannerContext, currentRow, offset, length);
5847             if (!moreRows) {
5848               return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
5849             }
5850             results.clear();
5851             continue;
5852           }
5853 
5854           // Ok, we are good, let's try to get some results from the main heap.
5855           populateResult(results, this.storeHeap, scannerContext, currentRow, offset, length);
5856 
5857           if (scannerContext.checkAnyLimitReached(LimitScope.BETWEEN_CELLS)) {
5858             if (hasFilterRow) {
5859               throw new IncompatibleFilterException(
5860                   "Filter whose hasFilterRow() returns true is incompatible with scans that must "
5861                       + " stop mid-row because of a limit. ScannerContext:" + scannerContext);
5862             }
5863             return true;
5864           }
5865 
5866           Cell nextKv = this.storeHeap.peek();
5867           stopRow = nextKv == null ||
5868               isStopRow(nextKv.getRowArray(), nextKv.getRowOffset(), nextKv.getRowLength());
5869           // save that the row was empty before filters applied to it.
5870           final boolean isEmptyRow = results.isEmpty();
5871 
5872           // We have the part of the row necessary for filtering (all of it, usually).
5873           // First filter with the filterRow(List).
5874           FilterWrapper.FilterRowRetCode ret = FilterWrapper.FilterRowRetCode.NOT_CALLED;
5875           if (hasFilterRow) {
5876             ret = filter.filterRowCellsWithRet(results);
5877 
5878             // We don't know how the results have changed after being filtered. Must set progress
5879             // according to contents of results now. However, a change in the results should not
5880             // affect the time progress. Thus preserve whatever time progress has been made
5881             long timeProgress = scannerContext.getTimeProgress();
5882             if (scannerContext.getKeepProgress()) {
5883               scannerContext.setProgress(initialBatchProgress, initialSizeProgress,
5884                 initialTimeProgress);
5885             } else {
5886               scannerContext.clearProgress();
5887             }
5888             scannerContext.setTimeProgress(timeProgress);
5889             scannerContext.incrementBatchProgress(results.size());
5890             for (Cell cell : results) {
5891               scannerContext.incrementSizeProgress(CellUtil.estimatedHeapSizeOfWithoutTags(cell));
5892             }
5893           }
5894 
5895           if (isEmptyRow || ret == FilterWrapper.FilterRowRetCode.EXCLUDE || filterRow()) {
5896             incrementCountOfRowsFilteredMetric(scannerContext);
5897             results.clear();
5898             boolean moreRows = nextRow(scannerContext, currentRow, offset, length);
5899             if (!moreRows) {
5900               return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
5901             }
5902 
5903             // This row was totally filtered out, if this is NOT the last row,
5904             // we should continue on. Otherwise, nothing else to do.
5905             if (!stopRow) continue;
5906             return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
5907           }
5908 
5909           // Ok, we are done with storeHeap for this row.
5910           // Now we may need to fetch additional, non-essential data into row.
5911           // These values are not needed for filter to work, so we postpone their
5912           // fetch to (possibly) reduce amount of data loads from disk.
5913           if (this.joinedHeap != null) {
5914             boolean mayHaveData = joinedHeapMayHaveData(currentRow, offset, length);
5915             if (mayHaveData) {
5916               joinedContinuationRow = current;
5917               populateFromJoinedHeap(results, scannerContext);
5918 
5919               if (scannerContext.checkAnyLimitReached(LimitScope.BETWEEN_CELLS)) {
5920                 return true;
5921               }
5922             }
5923           }
5924         } else {
5925           // Populating from the joined heap was stopped by limits, populate some more.
5926           populateFromJoinedHeap(results, scannerContext);
5927           if (scannerContext.checkAnyLimitReached(LimitScope.BETWEEN_CELLS)) {
5928             return true;
5929           }
5930         }
5931         // We may have just called populateFromJoinedMap and hit the limits. If that is
5932         // the case, we need to call it again on the next next() invocation.
5933         if (joinedContinuationRow != null) {
5934           return scannerContext.setScannerState(NextState.MORE_VALUES).hasMoreValues();
5935         }
5936 
5937         // Finally, we are done with both joinedHeap and storeHeap.
5938         // Double check to prevent empty rows from appearing in result. It could be
5939         // the case when SingleColumnValueExcludeFilter is used.
5940         if (results.isEmpty()) {
5941           incrementCountOfRowsFilteredMetric(scannerContext);
5942           boolean moreRows = nextRow(scannerContext, currentRow, offset, length);
5943           if (!moreRows) {
5944             return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
5945           }
5946           if (!stopRow) continue;
5947         }
5948 
5949         if (stopRow) {
5950           return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
5951         } else {
5952           return scannerContext.setScannerState(NextState.MORE_VALUES).hasMoreValues();
5953         }
5954       }
5955     }
5956 
5957     protected void incrementCountOfRowsFilteredMetric(ScannerContext scannerContext) {
5958       if (scannerContext == null || !scannerContext.isTrackingMetrics()) return;
5959 
5960       scannerContext.getMetrics().countOfRowsFiltered.incrementAndGet();
5961     }
5962 
5963     protected void incrementCountOfRowsScannedMetric(ScannerContext scannerContext) {
5964       if (scannerContext == null || !scannerContext.isTrackingMetrics()) return;
5965 
5966       scannerContext.getMetrics().countOfRowsScanned.incrementAndGet();
5967     }
5968 
5969     /**
5970      * @param currentRow
5971      * @param offset
5972      * @param length
5973      * @return true when the joined heap may have data for the current row
5974      * @throws IOException
5975      */
5976     private boolean joinedHeapMayHaveData(byte[] currentRow, int offset, short length)
5977         throws IOException {
5978       Cell nextJoinedKv = joinedHeap.peek();
5979       boolean matchCurrentRow =
5980           nextJoinedKv != null && CellUtil.matchingRow(nextJoinedKv, currentRow, offset, length);
5981       boolean matchAfterSeek = false;
5982 
5983       // If the next value in the joined heap does not match the current row, try to seek to the
5984       // correct row
5985       if (!matchCurrentRow) {
5986         Cell firstOnCurrentRow = KeyValueUtil.createFirstOnRow(currentRow, offset, length);
5987         boolean seekSuccessful = this.joinedHeap.requestSeek(firstOnCurrentRow, true, true);
5988         matchAfterSeek =
5989             seekSuccessful && joinedHeap.peek() != null
5990                 && CellUtil.matchingRow(joinedHeap.peek(), currentRow, offset, length);
5991       }
5992 
5993       return matchCurrentRow || matchAfterSeek;
5994     }
5995 
5996     /**
5997      * This function is to maintain backward compatibility for 0.94 filters. HBASE-6429 combines
5998      * both filterRow & filterRow(List<KeyValue> kvs) functions. While 0.94 code or older, it may
5999      * not implement hasFilterRow as HBase-6429 expects because 0.94 hasFilterRow() only returns
6000      * true when filterRow(List<KeyValue> kvs) is overridden not the filterRow(). Therefore, the
6001      * filterRow() will be skipped.
6002      */
6003     private boolean filterRow() throws IOException {
6004       // when hasFilterRow returns true, filter.filterRow() will be called automatically inside
6005       // filterRowCells(List<Cell> kvs) so we skip that scenario here.
6006       return filter != null && (!filter.hasFilterRow())
6007           && filter.filterRow();
6008     }
6009 
6010     private boolean filterRowKey(byte[] row, int offset, short length) throws IOException {
6011       return filter != null
6012           && filter.filterRowKey(row, offset, length);
6013     }
6014 
6015     protected boolean nextRow(ScannerContext scannerContext, byte[] currentRow, int offset,
6016         short length) throws IOException {
6017       assert this.joinedContinuationRow == null:
6018         "Trying to go to next row during joinedHeap read.";
6019       Cell next;
6020       while ((next = this.storeHeap.peek()) != null &&
6021              CellUtil.matchingRow(next, currentRow, offset, length)) {
6022         this.storeHeap.next(MOCKED_LIST);
6023       }
6024       resetFilters();
6025 
6026       // Calling the hook in CP which allows it to do a fast forward
6027       return this.region.getCoprocessorHost() == null
6028           || this.region.getCoprocessorHost()
6029               .postScannerFilterRow(this, currentRow, offset, length);
6030     }
6031 
6032     protected boolean isStopRow(byte[] currentRow, int offset, short length) {
6033       return currentRow == null ||
6034           (stopRow != null &&
6035           comparator.compareRows(stopRow, 0, stopRow.length,
6036             currentRow, offset, length) <= isScan);
6037     }
6038 
6039     @Override
6040     public synchronized void close() {
6041       if (storeHeap != null) {
6042         storeHeap.close();
6043         storeHeap = null;
6044       }
6045       if (joinedHeap != null) {
6046         joinedHeap.close();
6047         joinedHeap = null;
6048       }
6049       // no need to synchronize here.
6050       scannerReadPoints.remove(this);
6051       this.filterClosed = true;
6052     }
6053 
6054     KeyValueHeap getStoreHeapForTesting() {
6055       return storeHeap;
6056     }
6057 
6058     @Override
6059     public synchronized boolean reseek(byte[] row) throws IOException {
6060       if (row == null) {
6061         throw new IllegalArgumentException("Row cannot be null.");
6062       }
6063       boolean result = false;
6064       startRegionOperation();
6065       KeyValue kv = KeyValueUtil.createFirstOnRow(row);
6066       try {
6067         // use request seek to make use of the lazy seek option. See HBASE-5520
6068         result = this.storeHeap.requestSeek(kv, true, true);
6069         if (this.joinedHeap != null) {
6070           result = this.joinedHeap.requestSeek(kv, true, true) || result;
6071         }
6072       } catch (FileNotFoundException e) {
6073         throw handleFileNotFound(e);
6074       } finally {
6075         closeRegionOperation();
6076       }
6077       return result;
6078     }
6079 
6080     private IOException handleFileNotFound(FileNotFoundException fnfe) throws IOException {
6081       // tries to refresh the store files, otherwise shutdown the RS.
6082       // TODO: add support for abort() of a single region and trigger reassignment.
6083       try {
6084         region.refreshStoreFiles(true);
6085         return new IOException("unable to read store file");
6086       } catch (IOException e) {
6087         String msg = "a store file got lost: " + fnfe.getMessage();
6088         LOG.error(msg);
6089         LOG.error("unable to refresh store files", e);
6090         abortRegionServer(msg);
6091         return new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " closing");
6092       }
6093     }
6094 
6095     private void abortRegionServer(String msg) throws IOException {
6096       if (rsServices instanceof HRegionServer) {
6097         ((HRegionServer)rsServices).abort(msg);
6098       }
6099       throw new UnsupportedOperationException("not able to abort RS after: " + msg);
6100     }
6101   }
6102 
6103   // Utility methods
6104   /**
6105    * A utility method to create new instances of HRegion based on the
6106    * {@link HConstants#REGION_IMPL} configuration property.
6107    * @param tableDir qualified path of directory where region should be located,
6108    * usually the table directory.
6109    * @param wal The WAL is the outbound log for any updates to the HRegion
6110    * The wal file is a logfile from the previous execution that's
6111    * custom-computed for this HRegion. The HRegionServer computes and sorts the
6112    * appropriate wal info for this HRegion. If there is a previous file
6113    * (implying that the HRegion has been written-to before), then read it from
6114    * the supplied path.
6115    * @param fs is the filesystem.
6116    * @param conf is global configuration settings.
6117    * @param regionInfo - HRegionInfo that describes the region
6118    * is new), then read them from the supplied path.
6119    * @param htd the table descriptor
6120    * @return the new instance
6121    */
6122   static HRegion newHRegion(Path tableDir, WAL wal, FileSystem fs,
6123       Configuration conf, HRegionInfo regionInfo, final HTableDescriptor htd,
6124       RegionServerServices rsServices) {
6125     try {
6126       @SuppressWarnings("unchecked")
6127       Class<? extends HRegion> regionClass =
6128           (Class<? extends HRegion>) conf.getClass(HConstants.REGION_IMPL, HRegion.class);
6129 
6130       Constructor<? extends HRegion> c =
6131           regionClass.getConstructor(Path.class, WAL.class, FileSystem.class,
6132               Configuration.class, HRegionInfo.class, HTableDescriptor.class,
6133               RegionServerServices.class);
6134 
6135       return c.newInstance(tableDir, wal, fs, conf, regionInfo, htd, rsServices);
6136     } catch (Throwable e) {
6137       // todo: what should I throw here?
6138       throw new IllegalStateException("Could not instantiate a region instance.", e);
6139     }
6140   }
6141 
6142   /**
6143    * Convenience method creating new HRegions. Used by createTable and by the
6144    * bootstrap code in the HMaster constructor.
6145    * Note, this method creates an {@link WAL} for the created region. It
6146    * needs to be closed explicitly.  Use {@link HRegion#getWAL()} to get
6147    * access.  <b>When done with a region created using this method, you will
6148    * need to explicitly close the {@link WAL} it created too; it will not be
6149    * done for you.  Not closing the wal will leave at least a daemon thread
6150    * running.</b>  Call {@link #closeHRegion(HRegion)} and it will do
6151    * necessary cleanup for you.
6152    * @param info Info for region to create.
6153    * @param rootDir Root directory for HBase instance
6154    * @return new HRegion
6155    *
6156    * @throws IOException
6157    */
6158   public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
6159       final Configuration conf, final HTableDescriptor hTableDescriptor)
6160   throws IOException {
6161     return createHRegion(info, rootDir, conf, hTableDescriptor, null);
6162   }
6163 
6164   /**
6165    * This will do the necessary cleanup a call to
6166    * {@link #createHRegion(HRegionInfo, Path, Configuration, HTableDescriptor)}
6167    * requires.  This method will close the region and then close its
6168    * associated {@link WAL} file.  You can still use it if you call the other createHRegion,
6169    * the one that takes an {@link WAL} instance but don't be surprised by the
6170    * call to the {@link WAL#close()} on the {@link WAL} the
6171    * HRegion was carrying.
6172    * @throws IOException
6173    */
6174   public static void closeHRegion(final HRegion r) throws IOException {
6175     if (r == null) return;
6176     r.close();
6177     if (r.getWAL() == null) return;
6178     r.getWAL().close();
6179   }
6180 
6181   /**
6182    * Convenience method creating new HRegions. Used by createTable.
6183    * The {@link WAL} for the created region needs to be closed explicitly.
6184    * Use {@link HRegion#getWAL()} to get access.
6185    *
6186    * @param info Info for region to create.
6187    * @param rootDir Root directory for HBase instance
6188    * @param wal shared WAL
6189    * @param initialize - true to initialize the region
6190    * @return new HRegion
6191    *
6192    * @throws IOException
6193    */
6194   public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
6195                                       final Configuration conf,
6196                                       final HTableDescriptor hTableDescriptor,
6197                                       final WAL wal,
6198                                       final boolean initialize)
6199       throws IOException {
6200     return createHRegion(info, rootDir, conf, hTableDescriptor,
6201         wal, initialize, false);
6202   }
6203 
6204   /**
6205    * Convenience method creating new HRegions. Used by createTable.
6206    * The {@link WAL} for the created region needs to be closed
6207    * explicitly, if it is not null.
6208    * Use {@link HRegion#getWAL()} to get access.
6209    *
6210    * @param info Info for region to create.
6211    * @param rootDir Root directory for HBase instance
6212    * @param wal shared WAL
6213    * @param initialize - true to initialize the region
6214    * @param ignoreWAL - true to skip generate new wal if it is null, mostly for createTable
6215    * @return new HRegion
6216    * @throws IOException
6217    */
6218   public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
6219                                       final Configuration conf,
6220                                       final HTableDescriptor hTableDescriptor,
6221                                       final WAL wal,
6222                                       final boolean initialize, final boolean ignoreWAL)
6223       throws IOException {
6224       Path tableDir = FSUtils.getTableDir(rootDir, info.getTable());
6225       return createHRegion(info, rootDir, tableDir, conf, hTableDescriptor, wal, initialize,
6226           ignoreWAL);
6227   }
6228 
6229   /**
6230    * Convenience method creating new HRegions. Used by createTable.
6231    * The {@link WAL} for the created region needs to be closed
6232    * explicitly, if it is not null.
6233    * Use {@link HRegion#getWAL()} to get access.
6234    *
6235    * @param info Info for region to create.
6236    * @param rootDir Root directory for HBase instance
6237    * @param tableDir table directory
6238    * @param wal shared WAL
6239    * @param initialize - true to initialize the region
6240    * @param ignoreWAL - true to skip generate new wal if it is null, mostly for createTable
6241    * @return new HRegion
6242    * @throws IOException
6243    */
6244   public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
6245       final Path tableDir, final Configuration conf, final HTableDescriptor hTableDescriptor,
6246       final WAL wal, final boolean initialize, final boolean ignoreWAL)
6247   throws IOException {
6248     LOG.info("creating HRegion " + info.getTable().getNameAsString()
6249         + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
6250         " Table name == " + info.getTable().getNameAsString());
6251     FileSystem fs = FileSystem.get(conf);
6252     HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, info);
6253     WAL effectiveWAL = wal;
6254     if (wal == null && !ignoreWAL) {
6255       // TODO HBASE-11983 There'll be no roller for this wal?
6256       // The WAL subsystem will use the default rootDir rather than the passed in rootDir
6257       // unless I pass along via the conf.
6258       Configuration confForWAL = new Configuration(conf);
6259       confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
6260       effectiveWAL = (new WALFactory(confForWAL,
6261           Collections.<WALActionsListener>singletonList(new MetricsWAL()),
6262           "hregion-" + RandomStringUtils.randomNumeric(8))).
6263             getWAL(info.getEncodedNameAsBytes());
6264     }
6265     HRegion region = HRegion.newHRegion(tableDir,
6266         effectiveWAL, fs, conf, info, hTableDescriptor, null);
6267     if (initialize) region.initialize(null);
6268     return region;
6269   }
6270 
6271   public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
6272                                       final Configuration conf,
6273                                       final HTableDescriptor hTableDescriptor,
6274                                       final WAL wal)
6275     throws IOException {
6276     return createHRegion(info, rootDir, conf, hTableDescriptor, wal, true);
6277   }
6278 
6279 
6280   /**
6281    * Open a Region.
6282    * @param info Info for region to be opened.
6283    * @param wal WAL for region to use. This method will call
6284    * WAL#setSequenceNumber(long) passing the result of the call to
6285    * HRegion#getMinSequenceId() to ensure the wal id is properly kept
6286    * up.  HRegionStore does this every time it opens a new region.
6287    * @return new HRegion
6288    *
6289    * @throws IOException
6290    */
6291   public static HRegion openHRegion(final HRegionInfo info,
6292       final HTableDescriptor htd, final WAL wal,
6293       final Configuration conf)
6294   throws IOException {
6295     return openHRegion(info, htd, wal, conf, null, null);
6296   }
6297 
6298   /**
6299    * Open a Region.
6300    * @param info Info for region to be opened
6301    * @param htd the table descriptor
6302    * @param wal WAL for region to use. This method will call
6303    * WAL#setSequenceNumber(long) passing the result of the call to
6304    * HRegion#getMinSequenceId() to ensure the wal id is properly kept
6305    * up.  HRegionStore does this every time it opens a new region.
6306    * @param conf The Configuration object to use.
6307    * @param rsServices An interface we can request flushes against.
6308    * @param reporter An interface we can report progress against.
6309    * @return new HRegion
6310    *
6311    * @throws IOException
6312    */
6313   public static HRegion openHRegion(final HRegionInfo info,
6314     final HTableDescriptor htd, final WAL wal, final Configuration conf,
6315     final RegionServerServices rsServices,
6316     final CancelableProgressable reporter)
6317   throws IOException {
6318     return openHRegion(FSUtils.getRootDir(conf), info, htd, wal, conf, rsServices, reporter);
6319   }
6320 
6321   /**
6322    * Open a Region.
6323    * @param rootDir Root directory for HBase instance
6324    * @param info Info for region to be opened.
6325    * @param htd the table descriptor
6326    * @param wal WAL for region to use. This method will call
6327    * WAL#setSequenceNumber(long) passing the result of the call to
6328    * HRegion#getMinSequenceId() to ensure the wal id is properly kept
6329    * up.  HRegionStore does this every time it opens a new region.
6330    * @param conf The Configuration object to use.
6331    * @return new HRegion
6332    * @throws IOException
6333    */
6334   public static HRegion openHRegion(Path rootDir, final HRegionInfo info,
6335       final HTableDescriptor htd, final WAL wal, final Configuration conf)
6336   throws IOException {
6337     return openHRegion(rootDir, info, htd, wal, conf, null, null);
6338   }
6339 
6340   /**
6341    * Open a Region.
6342    * @param rootDir Root directory for HBase instance
6343    * @param info Info for region to be opened.
6344    * @param htd the table descriptor
6345    * @param wal WAL for region to use. This method will call
6346    * WAL#setSequenceNumber(long) passing the result of the call to
6347    * HRegion#getMinSequenceId() to ensure the wal id is properly kept
6348    * up.  HRegionStore does this every time it opens a new region.
6349    * @param conf The Configuration object to use.
6350    * @param rsServices An interface we can request flushes against.
6351    * @param reporter An interface we can report progress against.
6352    * @return new HRegion
6353    * @throws IOException
6354    */
6355   public static HRegion openHRegion(final Path rootDir, final HRegionInfo info,
6356       final HTableDescriptor htd, final WAL wal, final Configuration conf,
6357       final RegionServerServices rsServices,
6358       final CancelableProgressable reporter)
6359   throws IOException {
6360     FileSystem fs = null;
6361     if (rsServices != null) {
6362       fs = rsServices.getFileSystem();
6363     }
6364     if (fs == null) {
6365       fs = FileSystem.get(conf);
6366     }
6367     return openHRegion(conf, fs, rootDir, info, htd, wal, rsServices, reporter);
6368   }
6369 
6370   /**
6371    * Open a Region.
6372    * @param conf The Configuration object to use.
6373    * @param fs Filesystem to use
6374    * @param rootDir Root directory for HBase instance
6375    * @param info Info for region to be opened.
6376    * @param htd the table descriptor
6377    * @param wal WAL for region to use. This method will call
6378    * WAL#setSequenceNumber(long) passing the result of the call to
6379    * HRegion#getMinSequenceId() to ensure the wal id is properly kept
6380    * up.  HRegionStore does this every time it opens a new region.
6381    * @return new HRegion
6382    * @throws IOException
6383    */
6384   public static HRegion openHRegion(final Configuration conf, final FileSystem fs,
6385       final Path rootDir, final HRegionInfo info, final HTableDescriptor htd, final WAL wal)
6386       throws IOException {
6387     return openHRegion(conf, fs, rootDir, info, htd, wal, null, null);
6388   }
6389 
6390   /**
6391    * Open a Region.
6392    * @param conf The Configuration object to use.
6393    * @param fs Filesystem to use
6394    * @param rootDir Root directory for HBase instance
6395    * @param info Info for region to be opened.
6396    * @param htd the table descriptor
6397    * @param wal WAL for region to use. This method will call
6398    * WAL#setSequenceNumber(long) passing the result of the call to
6399    * HRegion#getMinSequenceId() to ensure the wal id is properly kept
6400    * up.  HRegionStore does this every time it opens a new region.
6401    * @param rsServices An interface we can request flushes against.
6402    * @param reporter An interface we can report progress against.
6403    * @return new HRegion
6404    * @throws IOException
6405    */
6406   public static HRegion openHRegion(final Configuration conf, final FileSystem fs,
6407       final Path rootDir, final HRegionInfo info, final HTableDescriptor htd, final WAL wal,
6408       final RegionServerServices rsServices, final CancelableProgressable reporter)
6409       throws IOException {
6410     Path tableDir = FSUtils.getTableDir(rootDir, info.getTable());
6411     return openHRegion(conf, fs, rootDir, tableDir, info, htd, wal, rsServices, reporter);
6412   }
6413 
6414   /**
6415    * Open a Region.
6416    * @param conf The Configuration object to use.
6417    * @param fs Filesystem to use
6418    * @param rootDir Root directory for HBase instance
6419    * @param info Info for region to be opened.
6420    * @param htd the table descriptor
6421    * @param wal WAL for region to use. This method will call
6422    * WAL#setSequenceNumber(long) passing the result of the call to
6423    * HRegion#getMinSequenceId() to ensure the wal id is properly kept
6424    * up.  HRegionStore does this every time it opens a new region.
6425    * @param rsServices An interface we can request flushes against.
6426    * @param reporter An interface we can report progress against.
6427    * @return new HRegion
6428    * @throws IOException
6429    */
6430   public static HRegion openHRegion(final Configuration conf, final FileSystem fs,
6431       final Path rootDir, final Path tableDir, final HRegionInfo info, final HTableDescriptor htd,
6432       final WAL wal, final RegionServerServices rsServices,
6433       final CancelableProgressable reporter)
6434       throws IOException {
6435     if (info == null) throw new NullPointerException("Passed region info is null");
6436     if (LOG.isDebugEnabled()) {
6437       LOG.debug("Opening region: " + info);
6438     }
6439     HRegion r = HRegion.newHRegion(tableDir, wal, fs, conf, info, htd, rsServices);
6440     return r.openHRegion(reporter);
6441   }
6442 
6443 
6444   /**
6445    * Useful when reopening a closed region (normally for unit tests)
6446    * @param other original object
6447    * @param reporter An interface we can report progress against.
6448    * @return new HRegion
6449    * @throws IOException
6450    */
6451   public static HRegion openHRegion(final HRegion other, final CancelableProgressable reporter)
6452       throws IOException {
6453     HRegionFileSystem regionFs = other.getRegionFileSystem();
6454     HRegion r = newHRegion(regionFs.getTableDir(), other.getWAL(), regionFs.getFileSystem(),
6455         other.baseConf, other.getRegionInfo(), other.getTableDesc(), null);
6456     return r.openHRegion(reporter);
6457   }
6458 
6459   public static Region openHRegion(final Region other, final CancelableProgressable reporter)
6460         throws IOException {
6461     return openHRegion((HRegion)other, reporter);
6462   }
6463 
6464   /**
6465    * Open HRegion.
6466    * Calls initialize and sets sequenceId.
6467    * @return Returns <code>this</code>
6468    * @throws IOException
6469    */
6470   protected HRegion openHRegion(final CancelableProgressable reporter)
6471   throws IOException {
6472     // Refuse to open the region if we are missing local compression support
6473     checkCompressionCodecs();
6474     // Refuse to open the region if encryption configuration is incorrect or
6475     // codec support is missing
6476     checkEncryption();
6477     // Refuse to open the region if a required class cannot be loaded
6478     checkClassLoading();
6479     this.openSeqNum = initialize(reporter);
6480     this.mvcc.advanceTo(openSeqNum);
6481     if (wal != null && getRegionServerServices() != null && !writestate.readOnly
6482         && !recovering) {
6483       // Only write the region open event marker to WAL if (1) we are not read-only
6484       // (2) dist log replay is off or we are not recovering. In case region is
6485       // recovering, the open event will be written at setRecovering(false)
6486       writeRegionOpenMarker(wal, openSeqNum);
6487     }
6488     return this;
6489   }
6490 
6491   public static void warmupHRegion(final HRegionInfo info,
6492       final HTableDescriptor htd, final WAL wal, final Configuration conf,
6493       final RegionServerServices rsServices,
6494       final CancelableProgressable reporter)
6495       throws IOException {
6496 
6497     if (info == null) throw new NullPointerException("Passed region info is null");
6498 
6499     if (LOG.isDebugEnabled()) {
6500       LOG.debug("HRegion.Warming up region: " + info);
6501     }
6502 
6503     Path rootDir = FSUtils.getRootDir(conf);
6504     Path tableDir = FSUtils.getTableDir(rootDir, info.getTable());
6505 
6506     FileSystem fs = null;
6507     if (rsServices != null) {
6508       fs = rsServices.getFileSystem();
6509     }
6510     if (fs == null) {
6511       fs = FileSystem.get(conf);
6512     }
6513 
6514     HRegion r = HRegion.newHRegion(tableDir, wal, fs, conf, info, htd, rsServices);
6515     r.initializeWarmup(reporter);
6516     r.close();
6517   }
6518 
6519 
6520   private void checkCompressionCodecs() throws IOException {
6521     for (HColumnDescriptor fam: this.htableDescriptor.getColumnFamilies()) {
6522       CompressionTest.testCompression(fam.getCompression());
6523       CompressionTest.testCompression(fam.getCompactionCompression());
6524     }
6525   }
6526 
6527   private void checkEncryption() throws IOException {
6528     for (HColumnDescriptor fam: this.htableDescriptor.getColumnFamilies()) {
6529       EncryptionTest.testEncryption(conf, fam.getEncryptionType(), fam.getEncryptionKey());
6530     }
6531   }
6532 
6533   private void checkClassLoading() throws IOException {
6534     RegionSplitPolicy.getSplitPolicyClass(this.htableDescriptor, conf);
6535     RegionCoprocessorHost.testTableCoprocessorAttrs(conf, this.htableDescriptor);
6536   }
6537 
6538   /**
6539    * Create a daughter region from given a temp directory with the region data.
6540    * @param hri Spec. for daughter region to open.
6541    * @throws IOException
6542    */
6543   HRegion createDaughterRegionFromSplits(final HRegionInfo hri) throws IOException {
6544     // Move the files from the temporary .splits to the final /table/region directory
6545     fs.commitDaughterRegion(hri);
6546 
6547     // Create the daughter HRegion instance
6548     HRegion r = HRegion.newHRegion(this.fs.getTableDir(), this.getWAL(), fs.getFileSystem(),
6549         this.getBaseConf(), hri, this.getTableDesc(), rsServices);
6550     r.readRequestsCount.set(this.getReadRequestsCount() / 2);
6551     r.writeRequestsCount.set(this.getWriteRequestsCount() / 2);
6552     return r;
6553   }
6554 
6555   /**
6556    * Create a merged region given a temp directory with the region data.
6557    * @param region_b another merging region
6558    * @return merged HRegion
6559    * @throws IOException
6560    */
6561   HRegion createMergedRegionFromMerges(final HRegionInfo mergedRegionInfo,
6562       final HRegion region_b) throws IOException {
6563     HRegion r = HRegion.newHRegion(this.fs.getTableDir(), this.getWAL(),
6564         fs.getFileSystem(), this.getBaseConf(), mergedRegionInfo,
6565         this.getTableDesc(), this.rsServices);
6566     r.readRequestsCount.set(this.getReadRequestsCount()
6567         + region_b.getReadRequestsCount());
6568     r.writeRequestsCount.set(this.getWriteRequestsCount()
6569 
6570         + region_b.getWriteRequestsCount());
6571     this.fs.commitMergedRegion(mergedRegionInfo);
6572     return r;
6573   }
6574 
6575   /**
6576    * Inserts a new region's meta information into the passed
6577    * <code>meta</code> region. Used by the HMaster bootstrap code adding
6578    * new table to hbase:meta table.
6579    *
6580    * @param meta hbase:meta HRegion to be updated
6581    * @param r HRegion to add to <code>meta</code>
6582    *
6583    * @throws IOException
6584    */
6585   // TODO remove since only test and merge use this
6586   public static void addRegionToMETA(final HRegion meta, final HRegion r) throws IOException {
6587     meta.checkResources();
6588     // The row key is the region name
6589     byte[] row = r.getRegionInfo().getRegionName();
6590     final long now = EnvironmentEdgeManager.currentTime();
6591     final List<Cell> cells = new ArrayList<Cell>(2);
6592     cells.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
6593       HConstants.REGIONINFO_QUALIFIER, now,
6594       r.getRegionInfo().toByteArray()));
6595     // Set into the root table the version of the meta table.
6596     cells.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
6597       HConstants.META_VERSION_QUALIFIER, now,
6598       Bytes.toBytes(HConstants.META_VERSION)));
6599     meta.put(row, HConstants.CATALOG_FAMILY, cells);
6600   }
6601 
6602   /**
6603    * Computes the Path of the HRegion
6604    *
6605    * @param tabledir qualified path for table
6606    * @param name ENCODED region name
6607    * @return Path of HRegion directory
6608    * @deprecated For tests only; to be removed.
6609    */
6610   @Deprecated
6611   public static Path getRegionDir(final Path tabledir, final String name) {
6612     return new Path(tabledir, name);
6613   }
6614 
6615   /**
6616    * Computes the Path of the HRegion
6617    *
6618    * @param rootdir qualified path of HBase root directory
6619    * @param info HRegionInfo for the region
6620    * @return qualified path of region directory
6621    * @deprecated For tests only; to be removed.
6622    */
6623   @Deprecated
6624   @VisibleForTesting
6625   public static Path getRegionDir(final Path rootdir, final HRegionInfo info) {
6626     return new Path(
6627       FSUtils.getTableDir(rootdir, info.getTable()), info.getEncodedName());
6628   }
6629 
6630   /**
6631    * Determines if the specified row is within the row range specified by the
6632    * specified HRegionInfo
6633    *
6634    * @param info HRegionInfo that specifies the row range
6635    * @param row row to be checked
6636    * @return true if the row is within the range specified by the HRegionInfo
6637    */
6638   public static boolean rowIsInRange(HRegionInfo info, final byte [] row) {
6639     return ((info.getStartKey().length == 0) ||
6640         (Bytes.compareTo(info.getStartKey(), row) <= 0)) &&
6641         ((info.getEndKey().length == 0) ||
6642             (Bytes.compareTo(info.getEndKey(), row) > 0));
6643   }
6644 
6645   public static boolean rowIsInRange(HRegionInfo info, final byte [] row, final int offset,
6646       final short length) {
6647     return ((info.getStartKey().length == 0) ||
6648         (Bytes.compareTo(info.getStartKey(), 0, info.getStartKey().length,
6649           row, offset, length) <= 0)) &&
6650         ((info.getEndKey().length == 0) ||
6651           (Bytes.compareTo(info.getEndKey(), 0, info.getEndKey().length, row, offset, length) > 0));
6652   }
6653 
6654   /**
6655    * Merge two HRegions.  The regions must be adjacent and must not overlap.
6656    *
6657    * @return new merged HRegion
6658    * @throws IOException
6659    */
6660   public static HRegion mergeAdjacent(final HRegion srcA, final HRegion srcB)
6661   throws IOException {
6662     HRegion a = srcA;
6663     HRegion b = srcB;
6664 
6665     // Make sure that srcA comes first; important for key-ordering during
6666     // write of the merged file.
6667     if (srcA.getRegionInfo().getStartKey() == null) {
6668       if (srcB.getRegionInfo().getStartKey() == null) {
6669         throw new IOException("Cannot merge two regions with null start key");
6670       }
6671       // A's start key is null but B's isn't. Assume A comes before B
6672     } else if ((srcB.getRegionInfo().getStartKey() == null) ||
6673       (Bytes.compareTo(srcA.getRegionInfo().getStartKey(),
6674         srcB.getRegionInfo().getStartKey()) > 0)) {
6675       a = srcB;
6676       b = srcA;
6677     }
6678 
6679     if (!(Bytes.compareTo(a.getRegionInfo().getEndKey(),
6680         b.getRegionInfo().getStartKey()) == 0)) {
6681       throw new IOException("Cannot merge non-adjacent regions");
6682     }
6683     return merge(a, b);
6684   }
6685 
6686   /**
6687    * Merge two regions whether they are adjacent or not.
6688    *
6689    * @param a region a
6690    * @param b region b
6691    * @return new merged region
6692    * @throws IOException
6693    */
6694   public static HRegion merge(final HRegion a, final HRegion b) throws IOException {
6695     if (!a.getRegionInfo().getTable().equals(b.getRegionInfo().getTable())) {
6696       throw new IOException("Regions do not belong to the same table");
6697     }
6698 
6699     FileSystem fs = a.getRegionFileSystem().getFileSystem();
6700     // Make sure each region's cache is empty
6701     a.flush(true);
6702     b.flush(true);
6703 
6704     // Compact each region so we only have one store file per family
6705     a.compact(true);
6706     if (LOG.isDebugEnabled()) {
6707       LOG.debug("Files for region: " + a);
6708       a.getRegionFileSystem().logFileSystemState(LOG);
6709     }
6710     b.compact(true);
6711     if (LOG.isDebugEnabled()) {
6712       LOG.debug("Files for region: " + b);
6713       b.getRegionFileSystem().logFileSystemState(LOG);
6714     }
6715 
6716     RegionMergeTransactionImpl rmt = new RegionMergeTransactionImpl(a, b, true);
6717     if (!rmt.prepare(null)) {
6718       throw new IOException("Unable to merge regions " + a + " and " + b);
6719     }
6720     HRegionInfo mergedRegionInfo = rmt.getMergedRegionInfo();
6721     LOG.info("starting merge of regions: " + a + " and " + b
6722         + " into new region " + mergedRegionInfo.getRegionNameAsString()
6723         + " with start key <"
6724         + Bytes.toStringBinary(mergedRegionInfo.getStartKey())
6725         + "> and end key <"
6726         + Bytes.toStringBinary(mergedRegionInfo.getEndKey()) + ">");
6727     HRegion dstRegion;
6728     try {
6729       dstRegion = (HRegion)rmt.execute(null, null);
6730     } catch (IOException ioe) {
6731       rmt.rollback(null, null);
6732       throw new IOException("Failed merging region " + a + " and " + b
6733           + ", and successfully rolled back");
6734     }
6735     dstRegion.compact(true);
6736 
6737     if (LOG.isDebugEnabled()) {
6738       LOG.debug("Files for new region");
6739       dstRegion.getRegionFileSystem().logFileSystemState(LOG);
6740     }
6741 
6742     if (dstRegion.getRegionFileSystem().hasReferences(dstRegion.getTableDesc())) {
6743       throw new IOException("Merged region " + dstRegion
6744           + " still has references after the compaction, is compaction canceled?");
6745     }
6746 
6747     // Archiving the 'A' region
6748     HFileArchiver.archiveRegion(a.getBaseConf(), fs, a.getRegionInfo());
6749     // Archiving the 'B' region
6750     HFileArchiver.archiveRegion(b.getBaseConf(), fs, b.getRegionInfo());
6751 
6752     LOG.info("merge completed. New region is " + dstRegion);
6753     return dstRegion;
6754   }
6755 
6756   @Override
6757   public Result get(final Get get) throws IOException {
6758     checkRow(get.getRow(), "Get");
6759     // Verify families are all valid
6760     if (get.hasFamilies()) {
6761       for (byte [] family: get.familySet()) {
6762         checkFamily(family);
6763       }
6764     } else { // Adding all families to scanner
6765       for (byte[] family: this.htableDescriptor.getFamiliesKeys()) {
6766         get.addFamily(family);
6767       }
6768     }
6769     List<Cell> results = get(get, true);
6770     boolean stale = this.getRegionInfo().getReplicaId() != 0;
6771     return Result.create(results, get.isCheckExistenceOnly() ? !results.isEmpty() : null, stale);
6772   }
6773 
6774   @Override
6775   public List<Cell> get(Get get, boolean withCoprocessor) throws IOException {
6776 
6777     List<Cell> results = new ArrayList<Cell>();
6778 
6779     // pre-get CP hook
6780     if (withCoprocessor && (coprocessorHost != null)) {
6781        if (coprocessorHost.preGet(get, results)) {
6782          return results;
6783        }
6784     }
6785 
6786     Scan scan = new Scan(get);
6787 
6788     RegionScanner scanner = null;
6789     try {
6790       scanner = getScanner(scan);
6791       scanner.next(results);
6792     } finally {
6793       if (scanner != null)
6794         scanner.close();
6795     }
6796 
6797     // post-get CP hook
6798     if (withCoprocessor && (coprocessorHost != null)) {
6799       coprocessorHost.postGet(get, results);
6800     }
6801 
6802     // do after lock
6803     if (this.metricsRegion != null) {
6804       long totalSize = 0L;
6805       for (Cell cell : results) {
6806         totalSize += CellUtil.estimatedSerializedSizeOf(cell);
6807       }
6808       this.metricsRegion.updateGet(totalSize);
6809     }
6810 
6811     return results;
6812   }
6813 
6814   @Override
6815   public void mutateRow(RowMutations rm) throws IOException {
6816     // Don't need nonces here - RowMutations only supports puts and deletes
6817     mutateRowsWithLocks(rm.getMutations(), Collections.singleton(rm.getRow()));
6818   }
6819 
6820   /**
6821    * Perform atomic mutations within the region w/o nonces.
6822    * See {@link #mutateRowsWithLocks(Collection, Collection, long, long)}
6823    */
6824   public void mutateRowsWithLocks(Collection<Mutation> mutations,
6825       Collection<byte[]> rowsToLock) throws IOException {
6826     mutateRowsWithLocks(mutations, rowsToLock, HConstants.NO_NONCE, HConstants.NO_NONCE);
6827   }
6828 
6829   /**
6830    * Perform atomic mutations within the region.
6831    * @param mutations The list of mutations to perform.
6832    * <code>mutations</code> can contain operations for multiple rows.
6833    * Caller has to ensure that all rows are contained in this region.
6834    * @param rowsToLock Rows to lock
6835    * @param nonceGroup Optional nonce group of the operation (client Id)
6836    * @param nonce Optional nonce of the operation (unique random id to ensure "more idempotence")
6837    * If multiple rows are locked care should be taken that
6838    * <code>rowsToLock</code> is sorted in order to avoid deadlocks.
6839    * @throws IOException
6840    */
6841   @Override
6842   public void mutateRowsWithLocks(Collection<Mutation> mutations,
6843       Collection<byte[]> rowsToLock, long nonceGroup, long nonce) throws IOException {
6844     MultiRowMutationProcessor proc = new MultiRowMutationProcessor(mutations, rowsToLock);
6845     processRowsWithLocks(proc, -1, nonceGroup, nonce);
6846   }
6847 
6848   /**
6849    * @return the current load statistics for the the region
6850    */
6851   public ClientProtos.RegionLoadStats getRegionStats() {
6852     if (!regionStatsEnabled) {
6853       return null;
6854     }
6855     ClientProtos.RegionLoadStats.Builder stats = ClientProtos.RegionLoadStats.newBuilder();
6856     stats.setMemstoreLoad((int) (Math.min(100, (this.memstoreSize.get() * 100) / this
6857         .memstoreFlushSize)));
6858     stats.setHeapOccupancy((int)rsServices.getHeapMemoryManager().getHeapOccupancyPercent()*100);
6859     stats.setCompactionPressure((int)rsServices.getCompactionPressure()*100 > 100 ? 100 :
6860                 (int)rsServices.getCompactionPressure()*100);
6861     return stats.build();
6862   }
6863 
6864   @Override
6865   public void processRowsWithLocks(RowProcessor<?,?> processor) throws IOException {
6866     processRowsWithLocks(processor, rowProcessorTimeout, HConstants.NO_NONCE,
6867       HConstants.NO_NONCE);
6868   }
6869 
6870   @Override
6871   public void processRowsWithLocks(RowProcessor<?,?> processor, long nonceGroup, long nonce)
6872       throws IOException {
6873     processRowsWithLocks(processor, rowProcessorTimeout, nonceGroup, nonce);
6874   }
6875 
6876   @Override
6877   public void processRowsWithLocks(RowProcessor<?,?> processor, long timeout,
6878       long nonceGroup, long nonce) throws IOException {
6879 
6880     for (byte[] row : processor.getRowsToLock()) {
6881       checkRow(row, "processRowsWithLocks");
6882     }
6883     if (!processor.readOnly()) {
6884       checkReadOnly();
6885     }
6886     checkResources();
6887 
6888     startRegionOperation();
6889     WALEdit walEdit = new WALEdit();
6890 
6891     // 1. Run pre-process hook
6892     try {
6893       processor.preProcess(this, walEdit);
6894     } catch (IOException e) {
6895       closeRegionOperation();
6896       throw e;
6897     }
6898     // Short circuit the read only case
6899     if (processor.readOnly()) {
6900       try {
6901         long now = EnvironmentEdgeManager.currentTime();
6902         doProcessRowWithTimeout(
6903             processor, now, this, null, null, timeout);
6904         processor.postProcess(this, walEdit, true);
6905       } finally {
6906         closeRegionOperation();
6907       }
6908       return;
6909     }
6910 
6911     MultiVersionConcurrencyControl.WriteEntry writeEntry = null;
6912     boolean locked;
6913     boolean walSyncSuccessful = false;
6914     List<RowLock> acquiredRowLocks;
6915     long addedSize = 0;
6916     List<Mutation> mutations = new ArrayList<Mutation>();
6917     Collection<byte[]> rowsToLock = processor.getRowsToLock();
6918     long mvccNum = 0;
6919     WALKey walKey = null;
6920     try {
6921       // 2. Acquire the row lock(s)
6922       acquiredRowLocks = new ArrayList<RowLock>(rowsToLock.size());
6923       for (byte[] row : rowsToLock) {
6924         // Attempt to lock all involved rows, throw if any lock times out
6925         // use a writer lock for mixed reads and writes
6926         acquiredRowLocks.add(getRowLock(row));
6927       }
6928       // 3. Region lock
6929       lock(this.updatesLock.readLock(), acquiredRowLocks.size() == 0 ? 1 : acquiredRowLocks.size());
6930       locked = true;
6931 
6932       long now = EnvironmentEdgeManager.currentTime();
6933       try {
6934         // 4. Let the processor scan the rows, generate mutations and add
6935         //    waledits
6936         doProcessRowWithTimeout(
6937             processor, now, this, mutations, walEdit, timeout);
6938 
6939         if (!mutations.isEmpty()) {
6940 
6941           // 5. Call the preBatchMutate hook
6942           processor.preBatchMutate(this, walEdit);
6943 
6944           long txid = 0;
6945           // 6. Append no sync
6946           if (!walEdit.isEmpty()) {
6947             // we use HLogKey here instead of WALKey directly to support legacy coprocessors.
6948             walKey = new HLogKey(this.getRegionInfo().getEncodedNameAsBytes(),
6949               this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now,
6950               processor.getClusterIds(), nonceGroup, nonce, mvcc);
6951             txid = this.wal.append(this.htableDescriptor, this.getRegionInfo(),
6952                 walKey, walEdit, false);
6953           }
6954           if(walKey == null){
6955             // since we use wal sequence Id as mvcc, for SKIP_WAL changes we need a "faked" WALEdit
6956             // to get a sequence id assigned which is done by FSWALEntry#stampRegionSequenceId
6957             walKey = this.appendEmptyEdit(this.wal);
6958           }
6959 
6960           // 7. Start mvcc transaction
6961           writeEntry = walKey.getWriteEntry();
6962           mvccNum = walKey.getSequenceId();
6963 
6964 
6965 
6966           // 8. Apply to memstore
6967           for (Mutation m : mutations) {
6968             // Handle any tag based cell features
6969             rewriteCellTags(m.getFamilyCellMap(), m);
6970 
6971             for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) {
6972               Cell cell = cellScanner.current();
6973               CellUtil.setSequenceId(cell, mvccNum);
6974               Store store = getStore(cell);
6975               if (store == null) {
6976                 checkFamily(CellUtil.cloneFamily(cell));
6977                 // unreachable
6978               }
6979               addedSize += store.add(cell);
6980             }
6981           }
6982 
6983           // 9. Release region lock
6984           if (locked) {
6985             this.updatesLock.readLock().unlock();
6986             locked = false;
6987           }
6988 
6989           // 10. Release row lock(s)
6990           releaseRowLocks(acquiredRowLocks);
6991 
6992           // 11. Sync edit log
6993           if (txid != 0) {
6994             syncOrDefer(txid, getEffectiveDurability(processor.useDurability()));
6995           }
6996           walSyncSuccessful = true;
6997           // 12. call postBatchMutate hook
6998           processor.postBatchMutate(this);
6999         }
7000       } finally {
7001         // TODO: Make this method look like all other methods that are doing append/sync and
7002         // memstore rollback such as append and doMiniBatchMutation. Currently it is a little
7003         // different. Make them all share same code!
7004         if (!mutations.isEmpty() && !walSyncSuccessful) {
7005           LOG.warn("Wal sync failed. Roll back " + mutations.size() +
7006               " memstore keyvalues for row(s):" + StringUtils.byteToHexString(
7007               processor.getRowsToLock().iterator().next()) + "...");
7008           for (Mutation m : mutations) {
7009             for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) {
7010               Cell cell = cellScanner.current();
7011               getStore(cell).rollback(cell);
7012             }
7013           }
7014           if (writeEntry != null) {
7015             mvcc.complete(writeEntry);
7016             writeEntry = null;
7017           }
7018         }
7019         // 13. Roll mvcc forward
7020         if (writeEntry != null) {
7021           mvcc.completeAndWait(writeEntry);
7022         }
7023         if (locked) {
7024           this.updatesLock.readLock().unlock();
7025         }
7026         // release locks if some were acquired but another timed out
7027         releaseRowLocks(acquiredRowLocks);
7028       }
7029 
7030       // 14. Run post-process hook
7031       processor.postProcess(this, walEdit, walSyncSuccessful);
7032 
7033     } finally {
7034       closeRegionOperation();
7035       if (!mutations.isEmpty() &&
7036           isFlushSize(this.addAndGetGlobalMemstoreSize(addedSize))) {
7037         requestFlush();
7038       }
7039     }
7040   }
7041 
7042   private void doProcessRowWithTimeout(final RowProcessor<?,?> processor,
7043                                        final long now,
7044                                        final HRegion region,
7045                                        final List<Mutation> mutations,
7046                                        final WALEdit walEdit,
7047                                        final long timeout) throws IOException {
7048     // Short circuit the no time bound case.
7049     if (timeout < 0) {
7050       try {
7051         processor.process(now, region, mutations, walEdit);
7052       } catch (IOException e) {
7053         LOG.warn("RowProcessor:" + processor.getClass().getName() +
7054             " throws Exception on row(s):" +
7055             Bytes.toStringBinary(
7056               processor.getRowsToLock().iterator().next()) + "...", e);
7057         throw e;
7058       }
7059       return;
7060     }
7061 
7062     // Case with time bound
7063     FutureTask<Void> task =
7064       new FutureTask<Void>(new Callable<Void>() {
7065         @Override
7066         public Void call() throws IOException {
7067           try {
7068             processor.process(now, region, mutations, walEdit);
7069             return null;
7070           } catch (IOException e) {
7071             LOG.warn("RowProcessor:" + processor.getClass().getName() +
7072                 " throws Exception on row(s):" +
7073                 Bytes.toStringBinary(
7074                     processor.getRowsToLock().iterator().next()) + "...", e);
7075             throw e;
7076           }
7077         }
7078       });
7079     rowProcessorExecutor.execute(task);
7080     try {
7081       task.get(timeout, TimeUnit.MILLISECONDS);
7082     } catch (TimeoutException te) {
7083       LOG.error("RowProcessor timeout:" + timeout + " ms on row(s):" +
7084           Bytes.toStringBinary(processor.getRowsToLock().iterator().next()) +
7085           "...");
7086       throw new IOException(te);
7087     } catch (Exception e) {
7088       throw new IOException(e);
7089     }
7090   }
7091 
7092   /**
7093    * @param cell
7094    * @param tags
7095    * @return The passed-in List<Tag> but with the tags from <code>cell</code> added.
7096    */
7097   private static List<Tag> carryForwardTags(final Cell cell, final List<Tag> tags) {
7098     if (cell.getTagsLength() <= 0) return tags;
7099     List<Tag> newTags = tags == null? new ArrayList<Tag>(): /*Append Tags*/tags;
7100     Iterator<Tag> i =
7101         CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength());
7102     while (i.hasNext()) newTags.add(i.next());
7103     return newTags;
7104   }
7105 
7106   /**
7107    * Run a Get against passed in <code>store</code> on passed <code>row</code>, etc.
7108    * @param store
7109    * @param row
7110    * @param family
7111    * @param tr
7112    * @return Get result.
7113    * @throws IOException
7114    */
7115   private List<Cell> doGet(final Store store, final byte [] row,
7116       final Map.Entry<byte[], List<Cell>> family, final TimeRange tr)
7117   throws IOException {
7118     // Sort the cells so that they match the order that they
7119     // appear in the Get results. Otherwise, we won't be able to
7120     // find the existing values if the cells are not specified
7121     // in order by the client since cells are in an array list.
7122     Collections.sort(family.getValue(), store.getComparator());
7123     // Get previous values for all columns in this family
7124     Get get = new Get(row);
7125     for (Cell cell : family.getValue()) {
7126       get.addColumn(family.getKey(), CellUtil.cloneQualifier(cell));
7127     }
7128     if (tr != null) get.setTimeRange(tr.getMin(), tr.getMax());
7129     return get(get, false);
7130   }
7131 
7132   public Result append(Append append) throws IOException {
7133     return append(append, HConstants.NO_NONCE, HConstants.NO_NONCE);
7134   }
7135 
7136   // TODO: There's a lot of boiler plate code identical to increment.
7137   // We should refactor append and increment as local get-mutate-put
7138   // transactions, so all stores only go through one code path for puts.
7139 
7140   @Override
7141   public Result append(Append mutate, long nonceGroup, long nonce) throws IOException {
7142     Operation op = Operation.APPEND;
7143     byte[] row = mutate.getRow();
7144     checkRow(row, op.toString());
7145     boolean flush = false;
7146     Durability durability = getEffectiveDurability(mutate.getDurability());
7147     boolean writeToWAL = durability != Durability.SKIP_WAL;
7148     WALEdit walEdits = null;
7149     List<Cell> allKVs = new ArrayList<Cell>(mutate.size());
7150     Map<Store, List<Cell>> tempMemstore = new HashMap<Store, List<Cell>>();
7151     long size = 0;
7152     long txid = 0;
7153     checkReadOnly();
7154     checkResources();
7155     // Lock row
7156     startRegionOperation(op);
7157     this.writeRequestsCount.increment();
7158     RowLock rowLock = null;
7159     WALKey walKey = null;
7160     boolean doRollBackMemstore = false;
7161     try {
7162       rowLock = getRowLock(row);
7163       assert rowLock != null;
7164       try {
7165         lock(this.updatesLock.readLock());
7166         try {
7167           // Wait for all prior MVCC transactions to finish - while we hold the row lock
7168           // (so that we are guaranteed to see the latest state when we do our Get)
7169           mvcc.await();
7170           if (this.coprocessorHost != null) {
7171             Result r = this.coprocessorHost.preAppendAfterRowLock(mutate);
7172             if (r!= null) {
7173               return r;
7174             }
7175           }
7176           long now = EnvironmentEdgeManager.currentTime();
7177           // Process each family
7178           for (Map.Entry<byte[], List<Cell>> family : mutate.getFamilyCellMap().entrySet()) {
7179             Store store = stores.get(family.getKey());
7180             List<Cell> kvs = new ArrayList<Cell>(family.getValue().size());
7181 
7182             List<Cell> results = doGet(store, row, family, null);
7183 
7184             // Iterate the input columns and update existing values if they were
7185             // found, otherwise add new column initialized to the append value
7186 
7187             // Avoid as much copying as possible. We may need to rewrite and
7188             // consolidate tags. Bytes are only copied once.
7189             // Would be nice if KeyValue had scatter/gather logic
7190             int idx = 0;
7191             for (Cell cell : family.getValue()) {
7192               Cell newCell;
7193               Cell oldCell = null;
7194               if (idx < results.size()
7195                   && CellUtil.matchingQualifier(results.get(idx), cell)) {
7196                 oldCell = results.get(idx);
7197                 long ts = Math.max(now, oldCell.getTimestamp());
7198 
7199                 // Process cell tags
7200                 // Make a union of the set of tags in the old and new KVs
7201                 List<Tag> tags = Tag.carryForwardTags(null, oldCell);
7202                 tags = Tag.carryForwardTags(tags, cell);
7203                 tags = carryForwardTTLTag(tags, mutate);
7204 
7205                 newCell = getNewCell(row, ts, cell, oldCell, Tag.fromList(tags));
7206 
7207                 idx++;
7208               } else {
7209                 // Append's KeyValue.Type==Put and ts==HConstants.LATEST_TIMESTAMP
7210                 CellUtil.updateLatestStamp(cell, now);
7211 
7212                 // Cell TTL handling
7213                 newCell = getNewCell(mutate, cell);
7214               }
7215 
7216               // Give coprocessors a chance to update the new cell
7217               if (coprocessorHost != null) {
7218                 newCell = coprocessorHost.postMutationBeforeWAL(RegionObserver.MutationType.APPEND,
7219                     mutate, oldCell, newCell);
7220               }
7221               kvs.add(newCell);
7222 
7223               // Append update to WAL
7224               if (writeToWAL) {
7225                 if (walEdits == null) {
7226                   walEdits = new WALEdit();
7227                 }
7228                 walEdits.add(newCell);
7229               }
7230             }
7231 
7232             //store the kvs to the temporary memstore before writing WAL
7233             tempMemstore.put(store, kvs);
7234           }
7235 
7236           // Actually write to WAL now
7237           if (walEdits != null && !walEdits.isEmpty()) {
7238             if (writeToWAL) {
7239               // Using default cluster id, as this can only happen in the originating
7240               // cluster. A slave cluster receives the final value (not the delta)
7241               // as a Put.
7242               // we use HLogKey here instead of WALKey directly to support legacy coprocessors.
7243               walKey = new HLogKey(
7244                   getRegionInfo().getEncodedNameAsBytes(),
7245                   this.htableDescriptor.getTableName(),
7246                   WALKey.NO_SEQUENCE_ID,
7247                   nonceGroup,
7248                   nonce,
7249                   mvcc);
7250               txid =
7251                 this.wal.append(this.htableDescriptor, getRegionInfo(), walKey, walEdits, true);
7252             } else {
7253               recordMutationWithoutWal(mutate.getFamilyCellMap());
7254             }
7255           }
7256           if (walKey == null) {
7257             // Append a faked WALEdit in order for SKIP_WAL updates to get mvcc assigned
7258             walKey = this.appendEmptyEdit(this.wal);
7259           }
7260           // Do a get on the write entry... this will block until sequenceid is assigned... w/o it,
7261           // TestAtomicOperation fails.
7262           walKey.getWriteEntry();
7263 
7264           // Actually write to Memstore now
7265           if (!tempMemstore.isEmpty()) {
7266             for (Map.Entry<Store, List<Cell>> entry : tempMemstore.entrySet()) {
7267               Store store = entry.getKey();
7268               if (store.getFamily().getMaxVersions() == 1) {
7269                 // upsert if VERSIONS for this CF == 1
7270                 // Is this right? It immediately becomes visible? St.Ack 20150907
7271                 size += store.upsert(entry.getValue(), getSmallestReadPoint());
7272               } else {
7273                 // otherwise keep older versions around
7274                 for (Cell cell: entry.getValue()) {
7275                   // This stamping of sequenceid seems redundant; it is happening down in
7276                   // FSHLog when we consume edits off the ring buffer.
7277                   CellUtil.setSequenceId(cell, walKey.getWriteEntry().getWriteNumber());
7278                   size += store.add(cell);
7279                   doRollBackMemstore = true;
7280                 }
7281               }
7282               // We add to all KVs here whereas when doing increment, we do it
7283               // earlier... why?
7284               allKVs.addAll(entry.getValue());
7285             }
7286 
7287             size = this.addAndGetGlobalMemstoreSize(size);
7288             flush = isFlushSize(size);
7289           }
7290         } finally {
7291           this.updatesLock.readLock().unlock();
7292         }
7293 
7294       } finally {
7295         rowLock.release();
7296         rowLock = null;
7297       }
7298       // sync the transaction log outside the rowlock
7299       if(txid != 0){
7300         syncOrDefer(txid, durability);
7301       }
7302       doRollBackMemstore = false;
7303     } finally {
7304       if (rowLock != null) {
7305         rowLock.release();
7306       }
7307       // if the wal sync was unsuccessful, remove keys from memstore
7308       WriteEntry we = walKey != null? walKey.getWriteEntry(): null;
7309       if (doRollBackMemstore) {
7310         rollbackMemstore(allKVs);
7311         if (we != null) mvcc.complete(we);
7312       } else if (we != null) {
7313         mvcc.completeAndWait(we);
7314       }
7315 
7316       closeRegionOperation(op);
7317     }
7318 
7319     if (this.metricsRegion != null) {
7320       this.metricsRegion.updateAppend();
7321     }
7322 
7323     if (flush) {
7324       // Request a cache flush. Do it outside update lock.
7325       requestFlush();
7326     }
7327 
7328     return mutate.isReturnResults() ? Result.create(allKVs) : null;
7329   }
7330 
7331   private static Cell getNewCell(final byte [] row, final long ts, final Cell cell,
7332       final Cell oldCell, final byte [] tagBytes) {
7333     // allocate an empty cell once
7334     Cell newCell = new KeyValue(row.length, cell.getFamilyLength(),
7335         cell.getQualifierLength(), ts, KeyValue.Type.Put,
7336         oldCell.getValueLength() + cell.getValueLength(),
7337         tagBytes == null? 0: tagBytes.length);
7338     // copy in row, family, and qualifier
7339     System.arraycopy(cell.getRowArray(), cell.getRowOffset(),
7340       newCell.getRowArray(), newCell.getRowOffset(), cell.getRowLength());
7341     System.arraycopy(cell.getFamilyArray(), cell.getFamilyOffset(),
7342       newCell.getFamilyArray(), newCell.getFamilyOffset(),
7343       cell.getFamilyLength());
7344     System.arraycopy(cell.getQualifierArray(), cell.getQualifierOffset(),
7345       newCell.getQualifierArray(), newCell.getQualifierOffset(),
7346       cell.getQualifierLength());
7347     // copy in the value
7348     System.arraycopy(oldCell.getValueArray(), oldCell.getValueOffset(),
7349       newCell.getValueArray(), newCell.getValueOffset(),
7350       oldCell.getValueLength());
7351     System.arraycopy(cell.getValueArray(), cell.getValueOffset(),
7352       newCell.getValueArray(),
7353       newCell.getValueOffset() + oldCell.getValueLength(),
7354       cell.getValueLength());
7355     // Copy in tag data
7356     if (tagBytes != null) {
7357       System.arraycopy(tagBytes, 0, newCell.getTagsArray(), newCell.getTagsOffset(),
7358         tagBytes.length);
7359     }
7360     return newCell;
7361   }
7362 
7363   private static Cell getNewCell(final Mutation mutate, final Cell cell) {
7364     Cell newCell = null;
7365     if (mutate.getTTL() != Long.MAX_VALUE) {
7366       // Add the new TTL tag
7367       newCell = new KeyValue(cell.getRowArray(), cell.getRowOffset(),
7368           cell.getRowLength(),
7369         cell.getFamilyArray(), cell.getFamilyOffset(),
7370           cell.getFamilyLength(),
7371         cell.getQualifierArray(), cell.getQualifierOffset(),
7372           cell.getQualifierLength(),
7373         cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()),
7374         cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(),
7375         carryForwardTTLTag(mutate));
7376     } else {
7377       newCell = cell;
7378     }
7379     return newCell;
7380   }
7381 
7382   public Result increment(Increment increment) throws IOException {
7383     return increment(increment, HConstants.NO_NONCE, HConstants.NO_NONCE);
7384   }
7385 
7386   // TODO: There's a lot of boiler plate code identical to append.
7387   // We should refactor append and increment as local get-mutate-put
7388   // transactions, so all stores only go through one code path for puts.
7389 
7390   // They are subtley different in quiet a few ways. This came out only
7391   // after study. I am not sure that many of the differences are intentional.
7392   // TODO: St.Ack 20150907
7393 
7394   @Override
7395   public Result increment(Increment mutation, long nonceGroup, long nonce)
7396   throws IOException {
7397     Operation op = Operation.INCREMENT;
7398     checkReadOnly();
7399     checkResources();
7400     checkRow(mutation.getRow(), op.toString());
7401     checkFamilies(mutation.getFamilyCellMap().keySet());
7402     startRegionOperation(op);
7403     this.writeRequestsCount.increment();
7404     try {
7405       // Which Increment is it? Narrow increment-only consistency or slow (default) and general
7406       // row-wide consistency.
7407 
7408       // So, difference between fastAndNarrowConsistencyIncrement and slowButConsistentIncrement is
7409       // that the former holds the row lock until the sync completes; this allows us to reason that
7410       // there are no other writers afoot when we read the current increment value. The row lock
7411       // means that we do not need to wait on mvcc reads to catch up to writes before we proceed
7412       // with the read, the root of the slowdown seen in HBASE-14460. The fast-path also does not
7413       // wait on mvcc to complete before returning to the client. We also reorder the write so that
7414       // the update of memstore happens AFTER sync returns; i.e. the write pipeline does less
7415       // zigzagging now.
7416       //
7417       // See the comment on INCREMENT_FAST_BUT_NARROW_CONSISTENCY_KEY
7418       // for the constraints that apply when you take this code path; it is correct but only if
7419       // Increments are used mutating an Increment Cell; mixing concurrent Put+Delete and Increment
7420       // will yield indeterminate results.
7421       return doIncrement(mutation, nonceGroup, nonce);
7422     } finally {
7423       if (this.metricsRegion != null) this.metricsRegion.updateIncrement();
7424       closeRegionOperation(op);
7425     }
7426   }
7427 
7428   private Result doIncrement(Increment increment, long nonceGroup, long nonce) throws IOException {
7429     RowLock rowLock = null;
7430     WALKey walKey = null;
7431     boolean doRollBackMemstore = false;
7432     long accumulatedResultSize = 0;
7433     List<Cell> allKVs = new ArrayList<Cell>(increment.size());
7434     List<Cell> memstoreCells = new ArrayList<Cell>();
7435     Durability effectiveDurability = getEffectiveDurability(increment.getDurability());
7436     try {
7437       rowLock = getRowLock(increment.getRow());
7438       long txid = 0;
7439       try {
7440         lock(this.updatesLock.readLock());
7441         try {
7442           // Wait for all prior MVCC transactions to finish - while we hold the row lock
7443           // (so that we are guaranteed to see the latest increment)
7444           this.mvcc.await();
7445           if (this.coprocessorHost != null) {
7446             Result r = this.coprocessorHost.preIncrementAfterRowLock(increment);
7447             if (r != null) return r;
7448           }
7449           long now = EnvironmentEdgeManager.currentTime();
7450           final boolean writeToWAL = effectiveDurability != Durability.SKIP_WAL;
7451           WALEdit walEdits = null;
7452           // Process increments a Store/family at a time.
7453           // Accumulate edits for memstore to add later after we've added to WAL.
7454           Map<Store, List<Cell>> forMemStore = new HashMap<Store, List<Cell>>();
7455           for (Map.Entry<byte [], List<Cell>> entry: increment.getFamilyCellMap().entrySet()) {
7456             byte [] columnFamilyName = entry.getKey();
7457             List<Cell> increments = entry.getValue();
7458             Store store = this.stores.get(columnFamilyName);
7459             // Do increment for this store; be sure to 'sort' the increments first so increments
7460             // match order in which we get back current Cells when we get.
7461             List<Cell> results = applyIncrementsToColumnFamily(increment, columnFamilyName,
7462                 sort(increments, store.getComparator()), now,
7463                 MultiVersionConcurrencyControl.NO_WRITE_NUMBER, allKVs, null);
7464             if (!results.isEmpty()) {
7465               forMemStore.put(store, results);
7466               // Prepare WAL updates
7467               if (writeToWAL) {
7468                 if (walEdits == null) walEdits = new WALEdit();
7469                 walEdits.getCells().addAll(results);
7470               }
7471             }
7472           }
7473           // Actually write to WAL now. If walEdits is non-empty, we write the WAL.
7474           if (walEdits != null && !walEdits.isEmpty()) {
7475             // Using default cluster id, as this can only happen in the originating cluster.
7476             // A slave cluster receives the final value (not the delta) as a Put. We use HLogKey
7477             // here instead of WALKey directly to support legacy coprocessors.
7478             walKey = new HLogKey(this.getRegionInfo().getEncodedNameAsBytes(),
7479               this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, nonceGroup, nonce,
7480               getMVCC());
7481             txid =
7482               this.wal.append(this.htableDescriptor, this.getRegionInfo(), walKey, walEdits, true);
7483           } else {
7484             // Append a faked WALEdit in order for SKIP_WAL updates to get mvccNum assigned
7485             walKey = this.appendEmptyEdit(this.wal);
7486           }
7487           // Get WriteEntry. Will wait on assign of the sequence id.
7488           walKey.getWriteEntry();
7489 
7490           // Now write to memstore, a family at a time.
7491           for (Map.Entry<Store, List<Cell>> entry: forMemStore.entrySet()) {
7492             Store store = entry.getKey();
7493             List<Cell> results = entry.getValue();
7494             if (store.getFamily().getMaxVersions() == 1) {
7495               // Upsert if VERSIONS for this CF == 1
7496               accumulatedResultSize += store.upsert(results, getSmallestReadPoint());
7497               // TODO: St.Ack 20151222 Why no rollback in this case?
7498             } else {
7499               // Otherwise keep older versions around
7500               for (Cell cell: results) {
7501                 // Why we need this?
7502                 CellUtil.setSequenceId(cell, walKey.getWriteEntry().getWriteNumber());
7503                 accumulatedResultSize += store.add(cell);
7504                 doRollBackMemstore = true;
7505               }
7506             }
7507           }
7508         } finally {
7509           this.updatesLock.readLock().unlock();
7510         }
7511       } finally {
7512         rowLock.release();
7513         rowLock = null;
7514       }
7515       // sync the transaction log outside the rowlock
7516       if(txid != 0) {
7517         syncOrDefer(txid, durability);
7518       }
7519       doRollBackMemstore = false;
7520     } finally {
7521       if (rowLock != null) {
7522         rowLock.release();
7523       }
7524       // if the wal sync was unsuccessful, remove keys from memstore
7525       if (doRollBackMemstore) {
7526         rollbackMemstore(memstoreCells);
7527         if (walKey != null) mvcc.complete(walKey.getWriteEntry());
7528       } else {
7529         if (walKey != null) mvcc.completeAndWait(walKey.getWriteEntry());
7530       }
7531     }
7532 
7533     // Request a cache flush.  Do it outside update lock.
7534     if (isFlushSize(this.addAndGetGlobalMemstoreSize(accumulatedResultSize))) requestFlush();
7535     return increment.isReturnResults() ? Result.create(allKVs) : null;
7536   }
7537 
7538   /**
7539    * @return Sorted list of <code>cells</code> using <code>comparator</code>
7540    */
7541   private static List<Cell> sort(List<Cell> cells, final Comparator<Cell> comparator) {
7542     Collections.sort(cells, comparator);
7543     return cells;
7544   }
7545 
7546   /**
7547    * Apply increments to a column family.
7548    * @param sortedIncrements The passed in increments to apply MUST be sorted so that they match
7549    * the order that they appear in the Get results (get results will be sorted on return).
7550    * Otherwise, we won't be able to find the existing values if the cells are not specified in
7551    * order by the client since cells are in an array list.
7552    * @islation Isolation level to use when running the 'get'. Pass null for default.
7553    * @return Resulting increments after <code>sortedIncrements</code> have been applied to current
7554    * values (if any -- else passed increment is the final result).
7555    * @throws IOException
7556    */
7557   private List<Cell> applyIncrementsToColumnFamily(Increment increment, byte[] columnFamilyName,
7558       List<Cell> sortedIncrements, long now, long mvccNum, List<Cell> allKVs,
7559       final IsolationLevel isolation)
7560   throws IOException {
7561     List<Cell> results = new ArrayList<Cell>(sortedIncrements.size());
7562     byte [] row = increment.getRow();
7563     // Get previous values for all columns in this family
7564     List<Cell> currentValues =
7565         getIncrementCurrentValue(increment, columnFamilyName, sortedIncrements, isolation);
7566     // Iterate the input columns and update existing values if they were found, otherwise
7567     // add new column initialized to the increment amount
7568     int idx = 0;
7569     for (int i = 0; i < sortedIncrements.size(); i++) {
7570       Cell inc = sortedIncrements.get(i);
7571       long incrementAmount = getLongValue(inc);
7572       // If increment amount == 0, then don't write this Increment to the WAL.
7573       boolean writeBack = (incrementAmount != 0);
7574       // Carry forward any tags that might have been added by a coprocessor.
7575       List<Tag> tags = Tag.carryForwardTags(inc);
7576 
7577       Cell currentValue = null;
7578       long ts = now;
7579       if (idx < currentValues.size() && CellUtil.matchingQualifier(currentValues.get(idx), inc)) {
7580         currentValue = currentValues.get(idx);
7581         ts = Math.max(now, currentValue.getTimestamp());
7582         incrementAmount += getLongValue(currentValue);
7583         // Carry forward all tags
7584         tags = Tag.carryForwardTags(tags, currentValue);
7585         if (i < (sortedIncrements.size() - 1) &&
7586             !CellUtil.matchingQualifier(inc, sortedIncrements.get(i + 1))) idx++;
7587       }
7588 
7589       // Append new incremented KeyValue to list
7590       byte [] qualifier = CellUtil.cloneQualifier(inc);
7591       byte [] incrementAmountInBytes = Bytes.toBytes(incrementAmount);
7592       tags = carryForwardTTLTag(tags, increment);
7593 
7594       Cell newValue = new KeyValue(row, 0, row.length,
7595         columnFamilyName, 0, columnFamilyName.length,
7596         qualifier, 0, qualifier.length,
7597         ts, KeyValue.Type.Put,
7598         incrementAmountInBytes, 0, incrementAmountInBytes.length,
7599         tags);
7600 
7601       // Don't set an mvcc if none specified. The mvcc may be assigned later in case where we
7602       // write the memstore AFTER we sync our edit to the log.
7603       if (mvccNum != MultiVersionConcurrencyControl.NO_WRITE_NUMBER) {
7604         CellUtil.setSequenceId(newValue, mvccNum);
7605       }
7606 
7607       // Give coprocessors a chance to update the new cell
7608       if (coprocessorHost != null) {
7609         newValue = coprocessorHost.postMutationBeforeWAL(
7610             RegionObserver.MutationType.INCREMENT, increment, currentValue, newValue);
7611       }
7612       allKVs.add(newValue);
7613       if (writeBack) {
7614         results.add(newValue);
7615       }
7616     }
7617     return results;
7618   }
7619 
7620   /**
7621    * @return Get the long out of the passed in Cell
7622    * @throws DoNotRetryIOException
7623    */
7624   private static long getLongValue(final Cell cell) throws DoNotRetryIOException {
7625     int len = cell.getValueLength();
7626     if (len != Bytes.SIZEOF_LONG) {
7627       // throw DoNotRetryIOException instead of IllegalArgumentException
7628       throw new DoNotRetryIOException("Field is not a long, it's " + len + " bytes wide");
7629     }
7630     return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), len);
7631   }
7632 
7633   /**
7634    * Do a specific Get on passed <code>columnFamily</code> and column qualifiers
7635    * from <code>incrementCoordinates</code> only.
7636    * @param increment
7637    * @param columnFamily
7638    * @param incrementCoordinates
7639    * @return Return the Cells to Increment
7640    * @throws IOException
7641    */
7642   private List<Cell> getIncrementCurrentValue(final Increment increment, byte [] columnFamily,
7643       final List<Cell> increments, final IsolationLevel isolation)
7644   throws IOException {
7645     Get get = new Get(increment.getRow());
7646     if (isolation != null) get.setIsolationLevel(isolation);
7647     for (Cell cell: increments) {
7648       get.addColumn(columnFamily, CellUtil.cloneQualifier(cell));
7649     }
7650     TimeRange tr = increment.getTimeRange();
7651     if (tr != null) {
7652       get.setTimeRange(tr.getMin(), tr.getMax());
7653     }
7654     return get(get, false);
7655   }
7656 
7657   private static List<Tag> carryForwardTTLTag(final Mutation mutation) {
7658     return carryForwardTTLTag(null, mutation);
7659   }
7660 
7661   /**
7662    * @return Carry forward the TTL tag if the increment is carrying one
7663    */
7664   private static List<Tag> carryForwardTTLTag(final List<Tag> tagsOrNull,
7665       final Mutation mutation) {
7666     long ttl = mutation.getTTL();
7667     if (ttl == Long.MAX_VALUE) return tagsOrNull;
7668     List<Tag> tags = tagsOrNull;
7669     // If we are making the array in here, given we are the last thing checked, we'll be only thing
7670     // in the array so set its size to '1' (I saw this being done in earlier version of
7671     // tag-handling).
7672     if (tags == null) tags = new ArrayList<Tag>(1);
7673     tags.add(new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(ttl)));
7674     return tags;
7675   }
7676 
7677   //
7678   // New HBASE-880 Helpers
7679   //
7680 
7681   private void checkFamily(final byte [] family)
7682   throws NoSuchColumnFamilyException {
7683     if (!this.htableDescriptor.hasFamily(family)) {
7684       throw new NoSuchColumnFamilyException("Column family " +
7685           Bytes.toString(family) + " does not exist in region " + this
7686           + " in table " + this.htableDescriptor);
7687     }
7688   }
7689 
7690   public static final long FIXED_OVERHEAD = ClassSize.align(
7691       ClassSize.OBJECT +
7692       ClassSize.ARRAY +
7693       44 * ClassSize.REFERENCE + 3 * Bytes.SIZEOF_INT +
7694       (14 * Bytes.SIZEOF_LONG) +
7695       5 * Bytes.SIZEOF_BOOLEAN);
7696 
7697   // woefully out of date - currently missing:
7698   // 1 x HashMap - coprocessorServiceHandlers
7699   // 6 x Counter - numMutationsWithoutWAL, dataInMemoryWithoutWAL,
7700   //   checkAndMutateChecksPassed, checkAndMutateChecksFailed, readRequestsCount,
7701   //   writeRequestsCount
7702   // 1 x HRegion$WriteState - writestate
7703   // 1 x RegionCoprocessorHost - coprocessorHost
7704   // 1 x RegionSplitPolicy - splitPolicy
7705   // 1 x MetricsRegion - metricsRegion
7706   // 1 x MetricsRegionWrapperImpl - metricsRegionWrapper
7707   public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
7708       ClassSize.OBJECT + // closeLock
7709       (2 * ClassSize.ATOMIC_BOOLEAN) + // closed, closing
7710       (3 * ClassSize.ATOMIC_LONG) + // memStoreSize, numPutsWithoutWAL, dataInMemoryWithoutWAL
7711       (2 * ClassSize.CONCURRENT_HASHMAP) +  // lockedRows, scannerReadPoints
7712       WriteState.HEAP_SIZE + // writestate
7713       ClassSize.CONCURRENT_SKIPLISTMAP + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + // stores
7714       (2 * ClassSize.REENTRANT_LOCK) + // lock, updatesLock
7715       MultiVersionConcurrencyControl.FIXED_SIZE // mvcc
7716       + ClassSize.TREEMAP // maxSeqIdInStores
7717       + 2 * ClassSize.ATOMIC_INTEGER // majorInProgress, minorInProgress
7718       ;
7719 
7720   @Override
7721   public long heapSize() {
7722     long heapSize = DEEP_OVERHEAD;
7723     for (Store store : this.stores.values()) {
7724       heapSize += store.heapSize();
7725     }
7726     // this does not take into account row locks, recent flushes, mvcc entries, and more
7727     return heapSize;
7728   }
7729 
7730   /*
7731    * This method calls System.exit.
7732    * @param message Message to print out.  May be null.
7733    */
7734   private static void printUsageAndExit(final String message) {
7735     if (message != null && message.length() > 0) System.out.println(message);
7736     System.out.println("Usage: HRegion CATALOG_TABLE_DIR [major_compact]");
7737     System.out.println("Options:");
7738     System.out.println(" major_compact  Pass this option to major compact " +
7739       "passed region.");
7740     System.out.println("Default outputs scan of passed region.");
7741     System.exit(1);
7742   }
7743 
7744   @Override
7745   public boolean registerService(Service instance) {
7746     /*
7747      * No stacking of instances is allowed for a single service name
7748      */
7749     Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
7750     if (coprocessorServiceHandlers.containsKey(serviceDesc.getFullName())) {
7751       LOG.error("Coprocessor service " + serviceDesc.getFullName() +
7752               " already registered, rejecting request from " + instance
7753       );
7754       return false;
7755     }
7756 
7757     coprocessorServiceHandlers.put(serviceDesc.getFullName(), instance);
7758     if (LOG.isDebugEnabled()) {
7759       LOG.debug("Registered coprocessor service: region=" +
7760           Bytes.toStringBinary(getRegionInfo().getRegionName()) +
7761           " service=" + serviceDesc.getFullName());
7762     }
7763     return true;
7764   }
7765 
7766   @Override
7767   public Message execService(RpcController controller, CoprocessorServiceCall call)
7768       throws IOException {
7769     String serviceName = call.getServiceName();
7770     String methodName = call.getMethodName();
7771     if (!coprocessorServiceHandlers.containsKey(serviceName)) {
7772       throw new UnknownProtocolException(null,
7773           "No registered coprocessor service found for name "+serviceName+
7774           " in region "+Bytes.toStringBinary(getRegionInfo().getRegionName()));
7775     }
7776 
7777     Service service = coprocessorServiceHandlers.get(serviceName);
7778     Descriptors.ServiceDescriptor serviceDesc = service.getDescriptorForType();
7779     Descriptors.MethodDescriptor methodDesc = serviceDesc.findMethodByName(methodName);
7780     if (methodDesc == null) {
7781       throw new UnknownProtocolException(service.getClass(),
7782           "Unknown method "+methodName+" called on service "+serviceName+
7783               " in region "+Bytes.toStringBinary(getRegionInfo().getRegionName()));
7784     }
7785 
7786     Message.Builder builder = service.getRequestPrototype(methodDesc).newBuilderForType();
7787     ProtobufUtil.mergeFrom(builder, call.getRequest());
7788     Message request = builder.build();
7789 
7790     if (coprocessorHost != null) {
7791       request = coprocessorHost.preEndpointInvocation(service, methodName, request);
7792     }
7793 
7794     final Message.Builder responseBuilder =
7795         service.getResponsePrototype(methodDesc).newBuilderForType();
7796     service.callMethod(methodDesc, controller, request, new RpcCallback<Message>() {
7797       @Override
7798       public void run(Message message) {
7799         if (message != null) {
7800           responseBuilder.mergeFrom(message);
7801         }
7802       }
7803     });
7804 
7805     if (coprocessorHost != null) {
7806       coprocessorHost.postEndpointInvocation(service, methodName, request, responseBuilder);
7807     }
7808 
7809     IOException exception = ResponseConverter.getControllerException(controller);
7810     if (exception != null) {
7811       throw exception;
7812     }
7813 
7814     return responseBuilder.build();
7815   }
7816 
7817   /*
7818    * Process table.
7819    * Do major compaction or list content.
7820    * @throws IOException
7821    */
7822   private static void processTable(final FileSystem fs, final Path p,
7823       final WALFactory walFactory, final Configuration c,
7824       final boolean majorCompact)
7825   throws IOException {
7826     HRegion region;
7827     FSTableDescriptors fst = new FSTableDescriptors(c);
7828     // Currently expects tables have one region only.
7829     if (FSUtils.getTableName(p).equals(TableName.META_TABLE_NAME)) {
7830       final WAL wal = walFactory.getMetaWAL(
7831           HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes());
7832       region = HRegion.newHRegion(p, wal, fs, c,
7833         HRegionInfo.FIRST_META_REGIONINFO, fst.get(TableName.META_TABLE_NAME), null);
7834     } else {
7835       throw new IOException("Not a known catalog table: " + p.toString());
7836     }
7837     try {
7838       region.mvcc.advanceTo(region.initialize(null));
7839       if (majorCompact) {
7840         region.compact(true);
7841       } else {
7842         // Default behavior
7843         Scan scan = new Scan();
7844         // scan.addFamily(HConstants.CATALOG_FAMILY);
7845         RegionScanner scanner = region.getScanner(scan);
7846         try {
7847           List<Cell> kvs = new ArrayList<Cell>();
7848           boolean done;
7849           do {
7850             kvs.clear();
7851             done = scanner.next(kvs);
7852             if (kvs.size() > 0) LOG.info(kvs);
7853           } while (done);
7854         } finally {
7855           scanner.close();
7856         }
7857       }
7858     } finally {
7859       region.close();
7860     }
7861   }
7862 
7863   boolean shouldForceSplit() {
7864     return this.splitRequest;
7865   }
7866 
7867   byte[] getExplicitSplitPoint() {
7868     return this.explicitSplitPoint;
7869   }
7870 
7871   void forceSplit(byte[] sp) {
7872     // This HRegion will go away after the forced split is successful
7873     // But if a forced split fails, we need to clear forced split.
7874     this.splitRequest = true;
7875     if (sp != null) {
7876       this.explicitSplitPoint = sp;
7877     }
7878   }
7879 
7880   void clearSplit() {
7881     this.splitRequest = false;
7882     this.explicitSplitPoint = null;
7883   }
7884 
7885   /**
7886    * Give the region a chance to prepare before it is split.
7887    */
7888   protected void prepareToSplit() {
7889     // nothing
7890   }
7891 
7892   /**
7893    * Return the splitpoint. null indicates the region isn't splittable
7894    * If the splitpoint isn't explicitly specified, it will go over the stores
7895    * to find the best splitpoint. Currently the criteria of best splitpoint
7896    * is based on the size of the store.
7897    */
7898   public byte[] checkSplit() {
7899     // Can't split META
7900     if (this.getRegionInfo().isMetaTable() ||
7901         TableName.NAMESPACE_TABLE_NAME.equals(this.getRegionInfo().getTable())) {
7902       if (shouldForceSplit()) {
7903         LOG.warn("Cannot split meta region in HBase 0.20 and above");
7904       }
7905       return null;
7906     }
7907 
7908     // Can't split region which is in recovering state
7909     if (this.isRecovering()) {
7910       LOG.info("Cannot split region " + this.getRegionInfo().getEncodedName() + " in recovery.");
7911       return null;
7912     }
7913 
7914     if (!splitPolicy.shouldSplit()) {
7915       return null;
7916     }
7917 
7918     byte[] ret = splitPolicy.getSplitPoint();
7919 
7920     if (ret != null) {
7921       try {
7922         checkRow(ret, "calculated split");
7923       } catch (IOException e) {
7924         LOG.error("Ignoring invalid split", e);
7925         return null;
7926       }
7927     }
7928     return ret;
7929   }
7930 
7931   /**
7932    * @return The priority that this region should have in the compaction queue
7933    */
7934   public int getCompactPriority() {
7935     int count = Integer.MAX_VALUE;
7936     for (Store store : stores.values()) {
7937       count = Math.min(count, store.getCompactPriority());
7938     }
7939     return count;
7940   }
7941 
7942 
7943   /** @return the coprocessor host */
7944   @Override
7945   public RegionCoprocessorHost getCoprocessorHost() {
7946     return coprocessorHost;
7947   }
7948 
7949   /** @param coprocessorHost the new coprocessor host */
7950   public void setCoprocessorHost(final RegionCoprocessorHost coprocessorHost) {
7951     this.coprocessorHost = coprocessorHost;
7952   }
7953 
7954   @Override
7955   public void startRegionOperation() throws IOException {
7956     startRegionOperation(Operation.ANY);
7957   }
7958 
7959   @Override
7960   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="SF_SWITCH_FALLTHROUGH",
7961     justification="Intentional")
7962   public void startRegionOperation(Operation op) throws IOException {
7963     switch (op) {
7964     case GET:  // read operations
7965     case SCAN:
7966       checkReadsEnabled();
7967     case INCREMENT: // write operations
7968     case APPEND:
7969     case SPLIT_REGION:
7970     case MERGE_REGION:
7971     case PUT:
7972     case DELETE:
7973     case BATCH_MUTATE:
7974     case COMPACT_REGION:
7975       // when a region is in recovering state, no read, split or merge is allowed
7976       if (isRecovering() && (this.disallowWritesInRecovering ||
7977               (op != Operation.PUT && op != Operation.DELETE && op != Operation.BATCH_MUTATE))) {
7978         throw new RegionInRecoveryException(getRegionInfo().getRegionNameAsString() +
7979           " is recovering; cannot take reads");
7980       }
7981       break;
7982     default:
7983       break;
7984     }
7985     if (op == Operation.MERGE_REGION || op == Operation.SPLIT_REGION
7986         || op == Operation.COMPACT_REGION) {
7987       // split, merge or compact region doesn't need to check the closing/closed state or lock the
7988       // region
7989       return;
7990     }
7991     if (this.closing.get()) {
7992       throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closing");
7993     }
7994     lock(lock.readLock());
7995     if (this.closed.get()) {
7996       lock.readLock().unlock();
7997       throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closed");
7998     }
7999     try {
8000       if (coprocessorHost != null) {
8001         coprocessorHost.postStartRegionOperation(op);
8002       }
8003     } catch (Exception e) {
8004       lock.readLock().unlock();
8005       throw new IOException(e);
8006     }
8007   }
8008 
8009   @Override
8010   public void closeRegionOperation() throws IOException {
8011     closeRegionOperation(Operation.ANY);
8012   }
8013 
8014   /**
8015    * Closes the lock. This needs to be called in the finally block corresponding
8016    * to the try block of {@link #startRegionOperation(Operation)}
8017    * @throws IOException
8018    */
8019   public void closeRegionOperation(Operation operation) throws IOException {
8020     lock.readLock().unlock();
8021     if (coprocessorHost != null) {
8022       coprocessorHost.postCloseRegionOperation(operation);
8023     }
8024   }
8025 
8026   /**
8027    * This method needs to be called before any public call that reads or
8028    * modifies stores in bulk. It has to be called just before a try.
8029    * #closeBulkRegionOperation needs to be called in the try's finally block
8030    * Acquires a writelock and checks if the region is closing or closed.
8031    * @throws NotServingRegionException when the region is closing or closed
8032    * @throws RegionTooBusyException if failed to get the lock in time
8033    * @throws InterruptedIOException if interrupted while waiting for a lock
8034    */
8035   private void startBulkRegionOperation(boolean writeLockNeeded)
8036       throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
8037     if (this.closing.get()) {
8038       throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closing");
8039     }
8040     if (writeLockNeeded) lock(lock.writeLock());
8041     else lock(lock.readLock());
8042     if (this.closed.get()) {
8043       if (writeLockNeeded) lock.writeLock().unlock();
8044       else lock.readLock().unlock();
8045       throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closed");
8046     }
8047   }
8048 
8049   /**
8050    * Closes the lock. This needs to be called in the finally block corresponding
8051    * to the try block of #startRegionOperation
8052    */
8053   private void closeBulkRegionOperation(){
8054     if (lock.writeLock().isHeldByCurrentThread()) lock.writeLock().unlock();
8055     else lock.readLock().unlock();
8056   }
8057 
8058   /**
8059    * Update counters for number of puts without wal and the size of possible data loss.
8060    * These information are exposed by the region server metrics.
8061    */
8062   private void recordMutationWithoutWal(final Map<byte [], List<Cell>> familyMap) {
8063     numMutationsWithoutWAL.increment();
8064     if (numMutationsWithoutWAL.get() <= 1) {
8065       LOG.info("writing data to region " + this +
8066                " with WAL disabled. Data may be lost in the event of a crash.");
8067     }
8068 
8069     long mutationSize = 0;
8070     for (List<Cell> cells: familyMap.values()) {
8071       assert cells instanceof RandomAccess;
8072       int listSize = cells.size();
8073       for (int i=0; i < listSize; i++) {
8074         Cell cell = cells.get(i);
8075         // TODO we need include tags length also here.
8076         mutationSize += KeyValueUtil.keyLength(cell) + cell.getValueLength();
8077       }
8078     }
8079 
8080     dataInMemoryWithoutWAL.add(mutationSize);
8081   }
8082 
8083   private void lock(final Lock lock)
8084       throws RegionTooBusyException, InterruptedIOException {
8085     lock(lock, 1);
8086   }
8087 
8088   /**
8089    * Try to acquire a lock.  Throw RegionTooBusyException
8090    * if failed to get the lock in time. Throw InterruptedIOException
8091    * if interrupted while waiting for the lock.
8092    */
8093   private void lock(final Lock lock, final int multiplier)
8094       throws RegionTooBusyException, InterruptedIOException {
8095     try {
8096       final long waitTime = Math.min(maxBusyWaitDuration,
8097           busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier));
8098       if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) {
8099         throw new RegionTooBusyException(
8100             "failed to get a lock in " + waitTime + " ms. " +
8101                 "regionName=" + (this.getRegionInfo() == null ? "unknown" :
8102                 this.getRegionInfo().getRegionNameAsString()) +
8103                 ", server=" + (this.getRegionServerServices() == null ? "unknown" :
8104                 this.getRegionServerServices().getServerName()));
8105       }
8106     } catch (InterruptedException ie) {
8107       LOG.info("Interrupted while waiting for a lock");
8108       InterruptedIOException iie = new InterruptedIOException();
8109       iie.initCause(ie);
8110       throw iie;
8111     }
8112   }
8113 
8114   /**
8115    * Calls sync with the given transaction ID if the region's table is not
8116    * deferring it.
8117    * @param txid should sync up to which transaction
8118    * @throws IOException If anything goes wrong with DFS
8119    */
8120   private void syncOrDefer(long txid, Durability durability) throws IOException {
8121     if (this.getRegionInfo().isMetaRegion()) {
8122       this.wal.sync(txid);
8123     } else {
8124       switch(durability) {
8125       case USE_DEFAULT:
8126         // do what table defaults to
8127         if (shouldSyncWAL()) {
8128           this.wal.sync(txid);
8129         }
8130         break;
8131       case SKIP_WAL:
8132         // nothing do to
8133         break;
8134       case ASYNC_WAL:
8135         // nothing do to
8136         break;
8137       case SYNC_WAL:
8138       case FSYNC_WAL:
8139         // sync the WAL edit (SYNC and FSYNC treated the same for now)
8140         this.wal.sync(txid);
8141         break;
8142       default:
8143         throw new RuntimeException("Unknown durability " + durability);
8144       }
8145     }
8146   }
8147 
8148   /**
8149    * Check whether we should sync the wal from the table's durability settings
8150    */
8151   private boolean shouldSyncWAL() {
8152     return durability.ordinal() >  Durability.ASYNC_WAL.ordinal();
8153   }
8154 
8155   /**
8156    * A mocked list implementation - discards all updates.
8157    */
8158   private static final List<Cell> MOCKED_LIST = new AbstractList<Cell>() {
8159 
8160     @Override
8161     public void add(int index, Cell element) {
8162       // do nothing
8163     }
8164 
8165     @Override
8166     public boolean addAll(int index, Collection<? extends Cell> c) {
8167       return false; // this list is never changed as a result of an update
8168     }
8169 
8170     @Override
8171     public KeyValue get(int index) {
8172       throw new UnsupportedOperationException();
8173     }
8174 
8175     @Override
8176     public int size() {
8177       return 0;
8178     }
8179   };
8180 
8181   /**
8182    * Facility for dumping and compacting catalog tables.
8183    * Only does catalog tables since these are only tables we for sure know
8184    * schema on.  For usage run:
8185    * <pre>
8186    *   ./bin/hbase org.apache.hadoop.hbase.regionserver.HRegion
8187    * </pre>
8188    * @throws IOException
8189    */
8190   public static void main(String[] args) throws IOException {
8191     if (args.length < 1) {
8192       printUsageAndExit(null);
8193     }
8194     boolean majorCompact = false;
8195     if (args.length > 1) {
8196       if (!args[1].toLowerCase().startsWith("major")) {
8197         printUsageAndExit("ERROR: Unrecognized option <" + args[1] + ">");
8198       }
8199       majorCompact = true;
8200     }
8201     final Path tableDir = new Path(args[0]);
8202     final Configuration c = HBaseConfiguration.create();
8203     final FileSystem fs = FileSystem.get(c);
8204     final Path logdir = new Path(c.get("hbase.tmp.dir"));
8205     final String logname = "wal" + FSUtils.getTableName(tableDir) + System.currentTimeMillis();
8206 
8207     final Configuration walConf = new Configuration(c);
8208     FSUtils.setRootDir(walConf, logdir);
8209     final WALFactory wals = new WALFactory(walConf, null, logname);
8210     try {
8211       processTable(fs, tableDir, wals, c, majorCompact);
8212     } finally {
8213        wals.close();
8214        // TODO: is this still right?
8215        BlockCache bc = new CacheConfig(c).getBlockCache();
8216        if (bc != null) bc.shutdown();
8217     }
8218   }
8219 
8220   @Override
8221   public long getOpenSeqNum() {
8222     return this.openSeqNum;
8223   }
8224 
8225   @Override
8226   public Map<byte[], Long> getMaxStoreSeqId() {
8227     return this.maxSeqIdInStores;
8228   }
8229 
8230   @Override
8231   public long getOldestSeqIdOfStore(byte[] familyName) {
8232     return wal.getEarliestMemstoreSeqNum(getRegionInfo().getEncodedNameAsBytes(), familyName);
8233   }
8234 
8235   @Override
8236   public CompactionState getCompactionState() {
8237     boolean hasMajor = majorInProgress.get() > 0, hasMinor = minorInProgress.get() > 0;
8238     return (hasMajor ? (hasMinor ? CompactionState.MAJOR_AND_MINOR : CompactionState.MAJOR)
8239         : (hasMinor ? CompactionState.MINOR : CompactionState.NONE));
8240   }
8241 
8242   public void reportCompactionRequestStart(boolean isMajor){
8243     (isMajor ? majorInProgress : minorInProgress).incrementAndGet();
8244   }
8245 
8246   public void reportCompactionRequestEnd(boolean isMajor, int numFiles, long filesSizeCompacted) {
8247     int newValue = (isMajor ? majorInProgress : minorInProgress).decrementAndGet();
8248 
8249     // metrics
8250     compactionsFinished.incrementAndGet();
8251     compactionNumFilesCompacted.addAndGet(numFiles);
8252     compactionNumBytesCompacted.addAndGet(filesSizeCompacted);
8253 
8254     assert newValue >= 0;
8255   }
8256 
8257   /**
8258    * Do not change this sequence id.
8259    * @return sequenceId
8260    */
8261   @VisibleForTesting
8262   public long getSequenceId() {
8263     return this.mvcc.getReadPoint();
8264   }
8265 
8266 
8267   /**
8268    * Append a faked WALEdit in order to get a long sequence number and wal syncer will just ignore
8269    * the WALEdit append later.
8270    * @param wal
8271    * @return Return the key used appending with no sync and no append.
8272    * @throws IOException
8273    */
8274   private WALKey appendEmptyEdit(final WAL wal) throws IOException {
8275     // we use HLogKey here instead of WALKey directly to support legacy coprocessors.
8276     @SuppressWarnings("deprecation")
8277     WALKey key = new HLogKey(getRegionInfo().getEncodedNameAsBytes(),
8278       getRegionInfo().getTable(), WALKey.NO_SEQUENCE_ID, 0, null,
8279       HConstants.NO_NONCE, HConstants.NO_NONCE, getMVCC());
8280 
8281     // Call append but with an empty WALEdit.  The returned sequence id will not be associated
8282     // with any edit and we can be sure it went in after all outstanding appends.
8283     try {
8284       wal.append(getTableDesc(), getRegionInfo(), key, WALEdit.EMPTY_WALEDIT, false);
8285     } catch (Throwable t) {
8286       // If exception, our mvcc won't get cleaned up by client, so do it here.
8287       getMVCC().complete(key.getWriteEntry());
8288     }
8289     return key;
8290   }
8291 
8292   /**
8293    * {@inheritDoc}
8294    */
8295   @Override
8296   public void onConfigurationChange(Configuration conf) {
8297     // Do nothing for now.
8298   }
8299 
8300   /**
8301    * {@inheritDoc}
8302    */
8303   @Override
8304   public void registerChildren(ConfigurationManager manager) {
8305     configurationManager = Optional.of(manager);
8306     for (Store s : this.stores.values()) {
8307       configurationManager.get().registerObserver(s);
8308     }
8309   }
8310 
8311   /**
8312    * {@inheritDoc}
8313    */
8314   @Override
8315   public void deregisterChildren(ConfigurationManager manager) {
8316     for (Store s : this.stores.values()) {
8317       configurationManager.get().deregisterObserver(s);
8318     }
8319   }
8320 
8321   /**
8322    * @return split policy for this region.
8323    */
8324   public RegionSplitPolicy getSplitPolicy() {
8325     return this.splitPolicy;
8326   }
8327 }