View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional information regarding
4    * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5    * "License"); you may not use this file except in compliance with the License. You may obtain a
6    * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
7    * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
8    * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
9    * for the specific language governing permissions and limitations under the License.
10   */
11  package org.apache.hadoop.hbase.namespace;
12  
13  import static org.junit.Assert.assertEquals;
14  import static org.junit.Assert.assertFalse;
15  import static org.junit.Assert.assertNotNull;
16  import static org.junit.Assert.assertNull;
17  import static org.junit.Assert.assertTrue;
18  import static org.junit.Assert.fail;
19  
20  import java.io.IOException;
21  import java.util.Collections;
22  import java.util.List;
23  import java.util.Set;
24  import java.util.concurrent.CountDownLatch;
25  
26  import org.apache.commons.lang.StringUtils;
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.CategoryBasedTimeout;
33  import org.apache.hadoop.hbase.Coprocessor;
34  import org.apache.hadoop.hbase.CoprocessorEnvironment;
35  import org.apache.hadoop.hbase.DoNotRetryIOException;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HTableDescriptor;
41  import org.apache.hadoop.hbase.MiniHBaseCluster;
42  import org.apache.hadoop.hbase.NamespaceDescriptor;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.Waiter;
45  import org.apache.hadoop.hbase.client.Connection;
46  import org.apache.hadoop.hbase.client.ConnectionFactory;
47  import org.apache.hadoop.hbase.client.HBaseAdmin;
48  import org.apache.hadoop.hbase.client.HTable;
49  import org.apache.hadoop.hbase.client.Mutation;
50  import org.apache.hadoop.hbase.client.RegionLocator;
51  import org.apache.hadoop.hbase.client.Table;
52  import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
54  import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
55  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
56  import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
57  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
58  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
59  import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
60  import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
61  import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
62  import org.apache.hadoop.hbase.master.HMaster;
63  import org.apache.hadoop.hbase.master.RegionState;
64  import org.apache.hadoop.hbase.master.RegionStates;
65  import org.apache.hadoop.hbase.master.TableNamespaceManager;
66  import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
67  import org.apache.hadoop.hbase.quotas.QuotaExceededException;
68  import org.apache.hadoop.hbase.quotas.QuotaUtil;
69  import org.apache.hadoop.hbase.regionserver.HRegion;
70  import org.apache.hadoop.hbase.regionserver.HRegionServer;
71  import org.apache.hadoop.hbase.regionserver.Region;
72  import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
73  import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
74  import org.apache.hadoop.hbase.testclassification.MediumTests;
75  import org.apache.hadoop.hbase.util.Bytes;
76  import org.apache.hadoop.hbase.util.FSUtils;
77  import org.apache.zookeeper.KeeperException;
78  import org.junit.After;
79  import org.junit.AfterClass;
80  import org.junit.BeforeClass;
81  import org.junit.Ignore;
82  import org.junit.Rule;
83  import org.junit.Test;
84  import org.junit.experimental.categories.Category;
85  import org.junit.rules.TestRule;
86  
87  import com.google.common.collect.Sets;
88  
89  @Category(MediumTests.class)
90  public class TestNamespaceAuditor {
91    @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
92        withTimeout(this.getClass()).withLookingForStuckThread(true).build();
93    private static final Log LOG = LogFactory.getLog(TestNamespaceAuditor.class);
94    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
95    private static HBaseAdmin ADMIN;
96    private String prefix = "TestNamespaceAuditor";
97  
98    @BeforeClass
99    public static void before() throws Exception {
100     UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
101     setupOnce();
102   }
103 
104   public static void setupOnce() throws Exception, IOException {
105     Configuration conf = UTIL.getConfiguration();
106     conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
107     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
108     conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
109     conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
110     conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
111       RegionServerObserver.class);
112     UTIL.startMiniCluster(1, 1);
113     waitForQuotaEnabled();
114     ADMIN = UTIL.getHBaseAdmin();
115   }
116 
117   @AfterClass
118   public static void tearDown() throws Exception {
119     UTIL.shutdownMiniCluster();
120   }
121 
122   @After
123   public void cleanup() throws Exception, KeeperException {
124     for (HTableDescriptor table : ADMIN.listTables()) {
125       ADMIN.disableTable(table.getTableName());
126       deleteTable(table.getTableName());
127     }
128     for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
129       if (ns.getName().startsWith(prefix)) {
130         ADMIN.deleteNamespace(ns.getName());
131       }
132     }
133     assertTrue("Quota manager not enabled", UTIL.getHBaseCluster().getMaster()
134         .getMasterQuotaManager().isQuotaEnabled());
135   }
136 
137   @Test
138   public void testTableOperations() throws Exception {
139     String nsp = prefix + "_np2";
140     NamespaceDescriptor nspDesc =
141         NamespaceDescriptor.create(nsp)
142             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
143             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
144     ADMIN.createNamespace(nspDesc);
145     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
146     assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
147     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
148 
149     HTableDescriptor tableDescOne =
150         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
151     tableDescOne.addFamily(fam1);
152     HTableDescriptor tableDescTwo =
153         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
154     tableDescTwo.addFamily(fam1);
155     HTableDescriptor tableDescThree =
156         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
157     tableDescThree.addFamily(fam1);
158     ADMIN.createTable(tableDescOne);
159     boolean constraintViolated = false;
160     try {
161       ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
162     } catch (Exception exp) {
163       assertTrue(exp instanceof IOException);
164       constraintViolated = true;
165     } finally {
166       assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
167         constraintViolated);
168     }
169     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
170     NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
171     assertNotNull(nspState);
172     assertTrue(nspState.getTables().size() == 2);
173     assertTrue(nspState.getRegionCount() == 5);
174     constraintViolated = false;
175     try {
176       ADMIN.createTable(tableDescThree);
177     } catch (Exception exp) {
178       assertTrue(exp instanceof IOException);
179       constraintViolated = true;
180     } finally {
181       assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
182         constraintViolated);
183     }
184   }
185 
186   @Test
187   public void testValidQuotas() throws Exception {
188     boolean exceptionCaught = false;
189     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
190     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
191     NamespaceDescriptor nspDesc =
192         NamespaceDescriptor.create(prefix + "vq1")
193             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
194             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
195     try {
196       ADMIN.createNamespace(nspDesc);
197     } catch (Exception exp) {
198       LOG.warn(exp);
199       exceptionCaught = true;
200     } finally {
201       assertTrue(exceptionCaught);
202       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
203     }
204     nspDesc =
205         NamespaceDescriptor.create(prefix + "vq2")
206             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
207             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
208     try {
209       ADMIN.createNamespace(nspDesc);
210     } catch (Exception exp) {
211       LOG.warn(exp);
212       exceptionCaught = true;
213     } finally {
214       assertTrue(exceptionCaught);
215       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
216     }
217     nspDesc =
218         NamespaceDescriptor.create(prefix + "vq3")
219             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
220             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
221     try {
222       ADMIN.createNamespace(nspDesc);
223     } catch (Exception exp) {
224       LOG.warn(exp);
225       exceptionCaught = true;
226     } finally {
227       assertTrue(exceptionCaught);
228       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
229     }
230     nspDesc =
231         NamespaceDescriptor.create(prefix + "vq4")
232             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
233             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
234     try {
235       ADMIN.createNamespace(nspDesc);
236     } catch (Exception exp) {
237       LOG.warn(exp);
238       exceptionCaught = true;
239     } finally {
240       assertTrue(exceptionCaught);
241       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
242     }
243   }
244 
245   @Test
246   public void testDeleteTable() throws Exception {
247     String namespace = prefix + "_dummy";
248     NamespaceDescriptor nspDesc =
249         NamespaceDescriptor.create(namespace)
250             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
251             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
252     ADMIN.createNamespace(nspDesc);
253     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
254     NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
255     assertNotNull("Namespace state found null for " + namespace, stateInfo);
256     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
257     HTableDescriptor tableDescOne =
258         new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
259     tableDescOne.addFamily(fam1);
260     HTableDescriptor tableDescTwo =
261         new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
262     tableDescTwo.addFamily(fam1);
263     ADMIN.createTable(tableDescOne);
264     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
265     stateInfo = getNamespaceState(nspDesc.getName());
266     assertNotNull("Namespace state found to be null.", stateInfo);
267     assertEquals(2, stateInfo.getTables().size());
268     assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
269     assertEquals(6, stateInfo.getRegionCount());
270     ADMIN.disableTable(tableDescOne.getTableName());
271     deleteTable(tableDescOne.getTableName());
272     stateInfo = getNamespaceState(nspDesc.getName());
273     assertNotNull("Namespace state found to be null.", stateInfo);
274     assertEquals(5, stateInfo.getRegionCount());
275     assertEquals(1, stateInfo.getTables().size());
276     ADMIN.disableTable(tableDescTwo.getTableName());
277     deleteTable(tableDescTwo.getTableName());
278     ADMIN.deleteNamespace(namespace);
279     stateInfo = getNamespaceState(namespace);
280     assertNull("Namespace state not found to be null.", stateInfo);
281   }
282 
283   public static class CPRegionServerObserver extends BaseRegionServerObserver {
284     private volatile boolean shouldFailMerge = false;
285 
286     public void failMerge(boolean fail) {
287       shouldFailMerge = fail;
288     }
289 
290     private boolean triggered = false;
291 
292     public synchronized void waitUtilTriggered() throws InterruptedException {
293       while (!triggered) {
294         wait();
295       }
296     }
297 
298     @Override
299     public synchronized void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
300         Region regionA, Region regionB) throws IOException {
301       triggered = true;
302       notifyAll();
303       if (shouldFailMerge) {
304         throw new IOException("fail merge");
305       }
306     }
307   }
308 
309   @Test
310   public void testRegionMerge() throws Exception {
311     String nsp1 = prefix + "_regiontest";
312     NamespaceDescriptor nspDesc =
313         NamespaceDescriptor.create(nsp1)
314             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
315             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
316     ADMIN.createNamespace(nspDesc);
317     final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
318     byte[] columnFamily = Bytes.toBytes("info");
319     HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
320     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
321     final int initialRegions = 3;
322     ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
323     try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
324         Table table = connection.getTable(tableTwo)) {
325       UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
326     }
327     ADMIN.flush(tableTwo);
328     List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
329     Collections.sort(hris);
330     // merge the two regions
331     final Set<String> encodedRegionNamesToMerge =
332         Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
333     ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(),
334       false);
335     waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
336     hris = ADMIN.getTableRegions(tableTwo);
337     assertEquals(initialRegions - 1, hris.size());
338     Collections.sort(hris);
339 
340     final HRegionInfo hriToSplit = hris.get(1);
341     ADMIN.split(tableTwo, Bytes.toBytes("500"));
342 
343     UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
344 
345       @Override
346       public boolean evaluate() throws Exception {
347         RegionStates regionStates =
348             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
349         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
350           if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
351             return false;
352           }
353           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
354             return false;
355           }
356         }
357         return true;
358       }
359 
360       @Override
361       public String explainFailure() throws Exception {
362         RegionStates regionStates =
363             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
364         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
365           if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
366             return hriToSplit + " which is expected to be split is still online";
367           }
368           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
369             return hri + " is still in not opened";
370           }
371         }
372         return "Unknown";
373       }
374     });
375     hris = ADMIN.getTableRegions(tableTwo);
376     assertEquals(initialRegions, hris.size());
377     Collections.sort(hris);
378 
379     // fail region merge through Coprocessor hook
380     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
381     HRegionServer regionServer = cluster.getRegionServer(0);
382     RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
383     Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
384     CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
385     regionServerObserver.failMerge(true);
386     regionServerObserver.triggered = false;
387 
388     ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(),
389       false);
390     regionServerObserver.waitUtilTriggered();
391     hris = ADMIN.getTableRegions(tableTwo);
392     assertEquals(initialRegions, hris.size());
393     Collections.sort(hris);
394     // verify that we cannot split
395     HRegionInfo hriToSplit2 = hris.get(1);
396     ADMIN.split(tableTwo,
397       TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
398     waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
399     assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
400   }
401 
402   private void waitForMergeToComplete(final TableName tableTwo,
403       final Set<String> encodedRegionNamesToMerge) throws Exception {
404     UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
405 
406       @Override
407       public boolean evaluate() throws Exception {
408         RegionStates regionStates =
409             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
410         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
411           if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
412             return false;
413           }
414           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
415             return false;
416           }
417         }
418         return true;
419       }
420 
421       @Override
422       public String explainFailure() throws Exception {
423         RegionStates regionStates =
424             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
425         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
426           if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
427             return hri + " which is expected to be merged is still online";
428           }
429           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
430             return hri + " is still in not opened";
431           }
432         }
433         return "Unknown";
434       }
435     });
436   }
437 
438   @Ignore("Hangs on occasion waiting on countdown latch") @Test
439   public void testRegionOperations() throws Exception {
440     String nsp1 = prefix + "_regiontest";
441     NamespaceDescriptor nspDesc =
442         NamespaceDescriptor.create(nsp1)
443             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
444             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
445     ADMIN.createNamespace(nspDesc);
446     boolean constraintViolated = false;
447     final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
448     byte[] columnFamily = Bytes.toBytes("info");
449     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
450     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
451     NamespaceTableAndRegionInfo stateInfo;
452     try {
453       ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
454     } catch (Exception exp) {
455       assertTrue(exp instanceof DoNotRetryIOException);
456       LOG.info(exp);
457       constraintViolated = true;
458     } finally {
459       assertTrue(constraintViolated);
460     }
461     assertFalse(ADMIN.tableExists(tableOne));
462     // This call will pass.
463     ADMIN.createTable(tableDescOne);
464     Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
465     HTable htable = (HTable) connection.getTable(tableOne);
466     UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
467     ADMIN.flush(tableOne);
468     stateInfo = getNamespaceState(nsp1);
469     assertEquals(1, stateInfo.getTables().size());
470     assertEquals(1, stateInfo.getRegionCount());
471     restartMaster();
472     ADMIN.split(tableOne, Bytes.toBytes("500"));
473     HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
474     CustomObserver observer =
475         (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
476           CustomObserver.class.getName());
477     assertNotNull(observer);
478     observer.postSplit.await();
479     assertEquals(2, ADMIN.getTableRegions(tableOne).size());
480     actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
481     observer =
482         (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
483           CustomObserver.class.getName());
484     assertNotNull(observer);
485     ADMIN.split(
486       tableOne,
487       getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo()
488           .getEndKey()));
489     observer.postSplit.await();
490     // Make sure no regions have been added.
491     List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
492     assertEquals(2, hris.size());
493     assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
494 
495     htable.close();
496   }
497 
498   /*
499    * Create a table and make sure that the table creation fails after adding this table entry into
500    * namespace quota cache. Now correct the failure and recreate the table with same name.
501    * HBASE-13394
502    */
503   @Test
504   public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
505     String nsp1 = prefix + "_testRecreateTable";
506     NamespaceDescriptor nspDesc =
507         NamespaceDescriptor.create(nsp1)
508             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
509             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
510     ADMIN.createNamespace(nspDesc);
511     final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
512     byte[] columnFamily = Bytes.toBytes("info");
513     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
514     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
515     MasterSyncObserver.throwExceptionInPreCreateTableHandler = true;
516     try {
517       try {
518         ADMIN.createTable(tableDescOne);
519         fail("Table " + tableOne.toString() + "creation should fail.");
520       } catch (Exception exp) {
521         LOG.error(exp);
522       }
523       assertFalse(ADMIN.tableExists(tableOne));
524 
525       NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
526       assertEquals("First table creation failed in namespace so number of tables in namespace "
527           + "should be 0.", 0, nstate.getTables().size());
528 
529       MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
530       try {
531         ADMIN.createTable(tableDescOne);
532       } catch (Exception e) {
533         fail("Table " + tableOne.toString() + "creation should succeed.");
534         LOG.error(e);
535       }
536       assertTrue(ADMIN.tableExists(tableOne));
537       nstate = getNamespaceState(nsp1);
538       assertEquals("First table was created successfully so table size in namespace should "
539           + "be one now.", 1, nstate.getTables().size());
540     } finally {
541       MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
542       if (ADMIN.tableExists(tableOne)) {
543         ADMIN.disableTable(tableOne);
544         deleteTable(tableOne);
545       }
546       ADMIN.deleteNamespace(nsp1);
547     }
548   }
549 
550   private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
551       IOException {
552     return getQuotaManager().getState(namespace);
553   }
554 
555   byte[] getSplitKey(byte[] startKey, byte[] endKey) {
556     String skey = Bytes.toString(startKey);
557     int key;
558     if (StringUtils.isBlank(skey)) {
559       key = Integer.parseInt(Bytes.toString(endKey)) / 2;
560     } else {
561       key = (int) (Integer.parseInt(skey) * 1.5);
562     }
563     return Bytes.toBytes("" + key);
564   }
565 
566   public static class CustomObserver extends BaseRegionObserver {
567     volatile CountDownLatch postSplit;
568     volatile CountDownLatch preSplitBeforePONR;
569 
570     @Override
571     public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
572         throws IOException {
573       postSplit.countDown();
574     }
575 
576     @Override
577     public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
578         byte[] splitKey, List<Mutation> metaEntries) throws IOException {
579       preSplitBeforePONR.countDown();
580     }
581 
582     @Override
583     public void start(CoprocessorEnvironment e) throws IOException {
584       postSplit = new CountDownLatch(1);
585       preSplitBeforePONR = new CountDownLatch(1);
586     }
587   }
588 
589   @Test
590   public void testStatePreserve() throws Exception {
591     final String nsp1 = prefix + "_testStatePreserve";
592     NamespaceDescriptor nspDesc =
593         NamespaceDescriptor.create(nsp1)
594             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
595             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
596     ADMIN.createNamespace(nspDesc);
597     TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
598     TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
599     TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
600     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
601     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
602     tableDescOne.addFamily(fam1);
603     HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
604     tableDescTwo.addFamily(fam1);
605     HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
606     tableDescThree.addFamily(fam1);
607     ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
608     ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
609     ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
610     ADMIN.disableTable(tableThree);
611     deleteTable(tableThree);
612     // wait for chore to complete
613     UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
614       @Override
615       public boolean evaluate() throws Exception {
616         return (getNamespaceState(nsp1).getTables().size() == 2);
617       }
618     });
619     NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
620     restartMaster();
621     NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
622     assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
623         .getTables().size(), after.getTables().size());
624   }
625 
626   private static void waitForQuotaEnabled() throws Exception {
627     UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
628       @Override
629       public boolean evaluate() throws Exception {
630         HMaster master = UTIL.getHBaseCluster().getMaster();
631         if (master == null) {
632           return false;
633         }
634         MasterQuotaManager quotaManager = master.getMasterQuotaManager();
635         return quotaManager != null && quotaManager.isQuotaEnabled();
636       }
637     });
638   }
639 
640   private void restartMaster() throws Exception {
641     UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
642     UTIL.getHBaseCluster().waitOnMaster(0);
643     UTIL.getHBaseCluster().startMaster();
644     waitForQuotaEnabled();
645   }
646 
647   private NamespaceAuditor getQuotaManager() {
648     return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
649   }
650 
651   public static class MasterSyncObserver extends BaseMasterObserver {
652     volatile CountDownLatch tableDeletionLatch;
653     static boolean throwExceptionInPreCreateTableHandler;
654 
655     @Override
656     public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
657         TableName tableName) throws IOException {
658       tableDeletionLatch = new CountDownLatch(1);
659     }
660 
661     @Override
662     public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
663         TableName tableName) throws IOException {
664       tableDeletionLatch.countDown();
665     }
666 
667     @Override
668     public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
669         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
670       if (throwExceptionInPreCreateTableHandler) {
671         throw new IOException("Throw exception as it is demanded.");
672       }
673     }
674   }
675 
676   private void deleteTable(final TableName tableName) throws Exception {
677     // NOTE: We need a latch because admin is not sync,
678     // so the postOp coprocessor method may be called after the admin operation returned.
679     MasterSyncObserver observer =
680         (MasterSyncObserver) UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
681             .findCoprocessor(MasterSyncObserver.class.getName());
682     ADMIN.deleteTable(tableName);
683     observer.tableDeletionLatch.await();
684   }
685 
686   @Test(expected = QuotaExceededException.class)
687   public void testExceedTableQuotaInNamespace() throws Exception {
688     String nsp = prefix + "_testExceedTableQuotaInNamespace";
689     NamespaceDescriptor nspDesc =
690         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
691             .build();
692     ADMIN.createNamespace(nspDesc);
693     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
694     assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
695     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
696     HTableDescriptor tableDescOne =
697         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
698     tableDescOne.addFamily(fam1);
699     HTableDescriptor tableDescTwo =
700         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
701     tableDescTwo.addFamily(fam1);
702     ADMIN.createTable(tableDescOne);
703     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
704   }
705   
706   @Test(expected = QuotaExceededException.class)
707   public void testCloneSnapshotQuotaExceed() throws Exception {
708     String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
709     NamespaceDescriptor nspDesc =
710         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
711             .build();
712     ADMIN.createNamespace(nspDesc);
713     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
714     TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
715     TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
716     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
717     HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
718     tableDescOne.addFamily(fam1);
719     ADMIN.createTable(tableDescOne);
720     String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
721     ADMIN.snapshot(snapshot, tableName);
722     ADMIN.cloneSnapshot(snapshot, cloneTableName);
723     ADMIN.deleteSnapshot(snapshot);
724   }
725 
726   @Test
727   public void testCloneSnapshot() throws Exception {
728     String nsp = prefix + "_testCloneSnapshot";
729     NamespaceDescriptor nspDesc =
730         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
731             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
732     ADMIN.createNamespace(nspDesc);
733     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
734     TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
735     TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
736 
737     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
738     HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
739     tableDescOne.addFamily(fam1);
740 
741     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
742     String snapshot = "snapshot_testCloneSnapshot";
743     ADMIN.snapshot(snapshot, tableName);
744     ADMIN.cloneSnapshot(snapshot, cloneTableName);
745 
746     int tableLength;
747     try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
748       tableLength = locator.getStartKeys().length;
749     }
750     assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
751 
752     try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
753       tableLength = locator.getStartKeys().length;
754     }
755     assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
756 
757     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
758     assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
759     assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
760 
761     ADMIN.deleteSnapshot(snapshot);
762   }
763 
764   @Test
765   public void testRestoreSnapshot() throws Exception {
766     String nsp = prefix + "_testRestoreSnapshot";
767     NamespaceDescriptor nspDesc =
768         NamespaceDescriptor.create(nsp)
769             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
770     ADMIN.createNamespace(nspDesc);
771     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
772     TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
773     HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
774     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
775     tableDescOne.addFamily(fam1);
776     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
777 
778     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
779     assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
780 
781     String snapshot = "snapshot_testRestoreSnapshot";
782     ADMIN.snapshot(snapshot, tableName1);
783 
784     List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
785     Collections.sort(regions);
786 
787     ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
788     Thread.sleep(2000);
789     assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
790 
791     ADMIN.disableTable(tableName1);
792     ADMIN.restoreSnapshot(snapshot);
793 
794     assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
795 
796     ADMIN.enableTable(tableName1);
797     ADMIN.deleteSnapshot(snapshot);
798   }
799 
800   @Test
801   public void testRestoreSnapshotQuotaExceed() throws Exception {
802     String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
803     NamespaceDescriptor nspDesc =
804         NamespaceDescriptor.create(nsp)
805             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
806     ADMIN.createNamespace(nspDesc);
807     NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
808     assertNotNull("Namespace descriptor found null.", ndesc);
809     TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
810     HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
811     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
812     tableDescOne.addFamily(fam1);
813 
814     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
815 
816     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
817     assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
818 
819     String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
820     ADMIN.snapshot(snapshot, tableName1);
821 
822     List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
823     Collections.sort(regions);
824 
825     ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
826     Thread.sleep(2000);
827     assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
828 
829     ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2");
830     ADMIN.modifyNamespace(ndesc);
831 
832     ADMIN.disableTable(tableName1);
833     try {
834       ADMIN.restoreSnapshot(snapshot);
835       fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
836           + " wraps IOException into RestoreSnapshotException");
837     } catch (RestoreSnapshotException ignore) {
838     }
839     ADMIN.enableTable(tableName1);
840     ADMIN.deleteSnapshot(snapshot);
841   }
842 }