1
2
3
4
5
6
7
8
9
10
11 package org.apache.hadoop.hbase.namespace;
12
13 import static org.junit.Assert.assertEquals;
14 import static org.junit.Assert.assertFalse;
15 import static org.junit.Assert.assertNotNull;
16 import static org.junit.Assert.assertNull;
17 import static org.junit.Assert.assertTrue;
18 import static org.junit.Assert.fail;
19
20 import java.io.IOException;
21 import java.util.Collections;
22 import java.util.List;
23 import java.util.Set;
24 import java.util.concurrent.CountDownLatch;
25
26 import org.apache.commons.lang.StringUtils;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.CategoryBasedTimeout;
33 import org.apache.hadoop.hbase.Coprocessor;
34 import org.apache.hadoop.hbase.CoprocessorEnvironment;
35 import org.apache.hadoop.hbase.DoNotRetryIOException;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.MiniHBaseCluster;
42 import org.apache.hadoop.hbase.NamespaceDescriptor;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.Waiter;
45 import org.apache.hadoop.hbase.client.Connection;
46 import org.apache.hadoop.hbase.client.ConnectionFactory;
47 import org.apache.hadoop.hbase.client.HBaseAdmin;
48 import org.apache.hadoop.hbase.client.HTable;
49 import org.apache.hadoop.hbase.client.Mutation;
50 import org.apache.hadoop.hbase.client.RegionLocator;
51 import org.apache.hadoop.hbase.client.Table;
52 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
54 import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
55 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
56 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
57 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
58 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
59 import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
60 import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
61 import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
62 import org.apache.hadoop.hbase.master.HMaster;
63 import org.apache.hadoop.hbase.master.RegionState;
64 import org.apache.hadoop.hbase.master.RegionStates;
65 import org.apache.hadoop.hbase.master.TableNamespaceManager;
66 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
67 import org.apache.hadoop.hbase.quotas.QuotaExceededException;
68 import org.apache.hadoop.hbase.quotas.QuotaUtil;
69 import org.apache.hadoop.hbase.regionserver.HRegion;
70 import org.apache.hadoop.hbase.regionserver.HRegionServer;
71 import org.apache.hadoop.hbase.regionserver.Region;
72 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
73 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
74 import org.apache.hadoop.hbase.testclassification.MediumTests;
75 import org.apache.hadoop.hbase.util.Bytes;
76 import org.apache.hadoop.hbase.util.FSUtils;
77 import org.apache.zookeeper.KeeperException;
78 import org.junit.After;
79 import org.junit.AfterClass;
80 import org.junit.BeforeClass;
81 import org.junit.Ignore;
82 import org.junit.Rule;
83 import org.junit.Test;
84 import org.junit.experimental.categories.Category;
85 import org.junit.rules.TestRule;
86
87 import com.google.common.collect.Sets;
88
89 @Category(MediumTests.class)
90 public class TestNamespaceAuditor {
91 @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
92 withTimeout(this.getClass()).withLookingForStuckThread(true).build();
93 private static final Log LOG = LogFactory.getLog(TestNamespaceAuditor.class);
94 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
95 private static HBaseAdmin ADMIN;
96 private String prefix = "TestNamespaceAuditor";
97
98 @BeforeClass
99 public static void before() throws Exception {
100 UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
101 setupOnce();
102 }
103
104 public static void setupOnce() throws Exception, IOException {
105 Configuration conf = UTIL.getConfiguration();
106 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
107 conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
108 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
109 conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
110 conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
111 RegionServerObserver.class);
112 UTIL.startMiniCluster(1, 1);
113 waitForQuotaEnabled();
114 ADMIN = UTIL.getHBaseAdmin();
115 }
116
117 @AfterClass
118 public static void tearDown() throws Exception {
119 UTIL.shutdownMiniCluster();
120 }
121
122 @After
123 public void cleanup() throws Exception, KeeperException {
124 for (HTableDescriptor table : ADMIN.listTables()) {
125 ADMIN.disableTable(table.getTableName());
126 deleteTable(table.getTableName());
127 }
128 for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
129 if (ns.getName().startsWith(prefix)) {
130 ADMIN.deleteNamespace(ns.getName());
131 }
132 }
133 assertTrue("Quota manager not enabled", UTIL.getHBaseCluster().getMaster()
134 .getMasterQuotaManager().isQuotaEnabled());
135 }
136
137 @Test
138 public void testTableOperations() throws Exception {
139 String nsp = prefix + "_np2";
140 NamespaceDescriptor nspDesc =
141 NamespaceDescriptor.create(nsp)
142 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
143 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
144 ADMIN.createNamespace(nspDesc);
145 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
146 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
147 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
148
149 HTableDescriptor tableDescOne =
150 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
151 tableDescOne.addFamily(fam1);
152 HTableDescriptor tableDescTwo =
153 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
154 tableDescTwo.addFamily(fam1);
155 HTableDescriptor tableDescThree =
156 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
157 tableDescThree.addFamily(fam1);
158 ADMIN.createTable(tableDescOne);
159 boolean constraintViolated = false;
160 try {
161 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
162 } catch (Exception exp) {
163 assertTrue(exp instanceof IOException);
164 constraintViolated = true;
165 } finally {
166 assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
167 constraintViolated);
168 }
169 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
170 NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
171 assertNotNull(nspState);
172 assertTrue(nspState.getTables().size() == 2);
173 assertTrue(nspState.getRegionCount() == 5);
174 constraintViolated = false;
175 try {
176 ADMIN.createTable(tableDescThree);
177 } catch (Exception exp) {
178 assertTrue(exp instanceof IOException);
179 constraintViolated = true;
180 } finally {
181 assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
182 constraintViolated);
183 }
184 }
185
186 @Test
187 public void testValidQuotas() throws Exception {
188 boolean exceptionCaught = false;
189 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
190 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
191 NamespaceDescriptor nspDesc =
192 NamespaceDescriptor.create(prefix + "vq1")
193 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
194 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
195 try {
196 ADMIN.createNamespace(nspDesc);
197 } catch (Exception exp) {
198 LOG.warn(exp);
199 exceptionCaught = true;
200 } finally {
201 assertTrue(exceptionCaught);
202 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
203 }
204 nspDesc =
205 NamespaceDescriptor.create(prefix + "vq2")
206 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
207 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
208 try {
209 ADMIN.createNamespace(nspDesc);
210 } catch (Exception exp) {
211 LOG.warn(exp);
212 exceptionCaught = true;
213 } finally {
214 assertTrue(exceptionCaught);
215 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
216 }
217 nspDesc =
218 NamespaceDescriptor.create(prefix + "vq3")
219 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
220 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
221 try {
222 ADMIN.createNamespace(nspDesc);
223 } catch (Exception exp) {
224 LOG.warn(exp);
225 exceptionCaught = true;
226 } finally {
227 assertTrue(exceptionCaught);
228 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
229 }
230 nspDesc =
231 NamespaceDescriptor.create(prefix + "vq4")
232 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
233 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
234 try {
235 ADMIN.createNamespace(nspDesc);
236 } catch (Exception exp) {
237 LOG.warn(exp);
238 exceptionCaught = true;
239 } finally {
240 assertTrue(exceptionCaught);
241 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
242 }
243 }
244
245 @Test
246 public void testDeleteTable() throws Exception {
247 String namespace = prefix + "_dummy";
248 NamespaceDescriptor nspDesc =
249 NamespaceDescriptor.create(namespace)
250 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
251 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
252 ADMIN.createNamespace(nspDesc);
253 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
254 NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
255 assertNotNull("Namespace state found null for " + namespace, stateInfo);
256 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
257 HTableDescriptor tableDescOne =
258 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
259 tableDescOne.addFamily(fam1);
260 HTableDescriptor tableDescTwo =
261 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
262 tableDescTwo.addFamily(fam1);
263 ADMIN.createTable(tableDescOne);
264 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
265 stateInfo = getNamespaceState(nspDesc.getName());
266 assertNotNull("Namespace state found to be null.", stateInfo);
267 assertEquals(2, stateInfo.getTables().size());
268 assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
269 assertEquals(6, stateInfo.getRegionCount());
270 ADMIN.disableTable(tableDescOne.getTableName());
271 deleteTable(tableDescOne.getTableName());
272 stateInfo = getNamespaceState(nspDesc.getName());
273 assertNotNull("Namespace state found to be null.", stateInfo);
274 assertEquals(5, stateInfo.getRegionCount());
275 assertEquals(1, stateInfo.getTables().size());
276 ADMIN.disableTable(tableDescTwo.getTableName());
277 deleteTable(tableDescTwo.getTableName());
278 ADMIN.deleteNamespace(namespace);
279 stateInfo = getNamespaceState(namespace);
280 assertNull("Namespace state not found to be null.", stateInfo);
281 }
282
283 public static class CPRegionServerObserver extends BaseRegionServerObserver {
284 private volatile boolean shouldFailMerge = false;
285
286 public void failMerge(boolean fail) {
287 shouldFailMerge = fail;
288 }
289
290 private boolean triggered = false;
291
292 public synchronized void waitUtilTriggered() throws InterruptedException {
293 while (!triggered) {
294 wait();
295 }
296 }
297
298 @Override
299 public synchronized void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
300 Region regionA, Region regionB) throws IOException {
301 triggered = true;
302 notifyAll();
303 if (shouldFailMerge) {
304 throw new IOException("fail merge");
305 }
306 }
307 }
308
309 @Test
310 public void testRegionMerge() throws Exception {
311 String nsp1 = prefix + "_regiontest";
312 NamespaceDescriptor nspDesc =
313 NamespaceDescriptor.create(nsp1)
314 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
315 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
316 ADMIN.createNamespace(nspDesc);
317 final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
318 byte[] columnFamily = Bytes.toBytes("info");
319 HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
320 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
321 final int initialRegions = 3;
322 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
323 try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
324 Table table = connection.getTable(tableTwo)) {
325 UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
326 }
327 ADMIN.flush(tableTwo);
328 List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
329 Collections.sort(hris);
330
331 final Set<String> encodedRegionNamesToMerge =
332 Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
333 ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(),
334 false);
335 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
336 hris = ADMIN.getTableRegions(tableTwo);
337 assertEquals(initialRegions - 1, hris.size());
338 Collections.sort(hris);
339
340 final HRegionInfo hriToSplit = hris.get(1);
341 ADMIN.split(tableTwo, Bytes.toBytes("500"));
342
343 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
344
345 @Override
346 public boolean evaluate() throws Exception {
347 RegionStates regionStates =
348 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
349 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
350 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
351 return false;
352 }
353 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
354 return false;
355 }
356 }
357 return true;
358 }
359
360 @Override
361 public String explainFailure() throws Exception {
362 RegionStates regionStates =
363 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
364 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
365 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
366 return hriToSplit + " which is expected to be split is still online";
367 }
368 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
369 return hri + " is still in not opened";
370 }
371 }
372 return "Unknown";
373 }
374 });
375 hris = ADMIN.getTableRegions(tableTwo);
376 assertEquals(initialRegions, hris.size());
377 Collections.sort(hris);
378
379
380 MiniHBaseCluster cluster = UTIL.getHBaseCluster();
381 HRegionServer regionServer = cluster.getRegionServer(0);
382 RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
383 Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
384 CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
385 regionServerObserver.failMerge(true);
386 regionServerObserver.triggered = false;
387
388 ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(),
389 false);
390 regionServerObserver.waitUtilTriggered();
391 hris = ADMIN.getTableRegions(tableTwo);
392 assertEquals(initialRegions, hris.size());
393 Collections.sort(hris);
394
395 HRegionInfo hriToSplit2 = hris.get(1);
396 ADMIN.split(tableTwo,
397 TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
398 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
399 assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
400 }
401
402 private void waitForMergeToComplete(final TableName tableTwo,
403 final Set<String> encodedRegionNamesToMerge) throws Exception {
404 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
405
406 @Override
407 public boolean evaluate() throws Exception {
408 RegionStates regionStates =
409 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
410 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
411 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
412 return false;
413 }
414 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
415 return false;
416 }
417 }
418 return true;
419 }
420
421 @Override
422 public String explainFailure() throws Exception {
423 RegionStates regionStates =
424 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
425 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
426 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
427 return hri + " which is expected to be merged is still online";
428 }
429 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
430 return hri + " is still in not opened";
431 }
432 }
433 return "Unknown";
434 }
435 });
436 }
437
438 @Ignore("Hangs on occasion waiting on countdown latch") @Test
439 public void testRegionOperations() throws Exception {
440 String nsp1 = prefix + "_regiontest";
441 NamespaceDescriptor nspDesc =
442 NamespaceDescriptor.create(nsp1)
443 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
444 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
445 ADMIN.createNamespace(nspDesc);
446 boolean constraintViolated = false;
447 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
448 byte[] columnFamily = Bytes.toBytes("info");
449 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
450 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
451 NamespaceTableAndRegionInfo stateInfo;
452 try {
453 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
454 } catch (Exception exp) {
455 assertTrue(exp instanceof DoNotRetryIOException);
456 LOG.info(exp);
457 constraintViolated = true;
458 } finally {
459 assertTrue(constraintViolated);
460 }
461 assertFalse(ADMIN.tableExists(tableOne));
462
463 ADMIN.createTable(tableDescOne);
464 Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
465 HTable htable = (HTable) connection.getTable(tableOne);
466 UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
467 ADMIN.flush(tableOne);
468 stateInfo = getNamespaceState(nsp1);
469 assertEquals(1, stateInfo.getTables().size());
470 assertEquals(1, stateInfo.getRegionCount());
471 restartMaster();
472 ADMIN.split(tableOne, Bytes.toBytes("500"));
473 HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
474 CustomObserver observer =
475 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
476 CustomObserver.class.getName());
477 assertNotNull(observer);
478 observer.postSplit.await();
479 assertEquals(2, ADMIN.getTableRegions(tableOne).size());
480 actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
481 observer =
482 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
483 CustomObserver.class.getName());
484 assertNotNull(observer);
485 ADMIN.split(
486 tableOne,
487 getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo()
488 .getEndKey()));
489 observer.postSplit.await();
490
491 List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
492 assertEquals(2, hris.size());
493 assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
494
495 htable.close();
496 }
497
498
499
500
501
502
503 @Test
504 public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
505 String nsp1 = prefix + "_testRecreateTable";
506 NamespaceDescriptor nspDesc =
507 NamespaceDescriptor.create(nsp1)
508 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
509 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
510 ADMIN.createNamespace(nspDesc);
511 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
512 byte[] columnFamily = Bytes.toBytes("info");
513 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
514 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
515 MasterSyncObserver.throwExceptionInPreCreateTableHandler = true;
516 try {
517 try {
518 ADMIN.createTable(tableDescOne);
519 fail("Table " + tableOne.toString() + "creation should fail.");
520 } catch (Exception exp) {
521 LOG.error(exp);
522 }
523 assertFalse(ADMIN.tableExists(tableOne));
524
525 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
526 assertEquals("First table creation failed in namespace so number of tables in namespace "
527 + "should be 0.", 0, nstate.getTables().size());
528
529 MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
530 try {
531 ADMIN.createTable(tableDescOne);
532 } catch (Exception e) {
533 fail("Table " + tableOne.toString() + "creation should succeed.");
534 LOG.error(e);
535 }
536 assertTrue(ADMIN.tableExists(tableOne));
537 nstate = getNamespaceState(nsp1);
538 assertEquals("First table was created successfully so table size in namespace should "
539 + "be one now.", 1, nstate.getTables().size());
540 } finally {
541 MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
542 if (ADMIN.tableExists(tableOne)) {
543 ADMIN.disableTable(tableOne);
544 deleteTable(tableOne);
545 }
546 ADMIN.deleteNamespace(nsp1);
547 }
548 }
549
550 private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
551 IOException {
552 return getQuotaManager().getState(namespace);
553 }
554
555 byte[] getSplitKey(byte[] startKey, byte[] endKey) {
556 String skey = Bytes.toString(startKey);
557 int key;
558 if (StringUtils.isBlank(skey)) {
559 key = Integer.parseInt(Bytes.toString(endKey)) / 2;
560 } else {
561 key = (int) (Integer.parseInt(skey) * 1.5);
562 }
563 return Bytes.toBytes("" + key);
564 }
565
566 public static class CustomObserver extends BaseRegionObserver {
567 volatile CountDownLatch postSplit;
568 volatile CountDownLatch preSplitBeforePONR;
569
570 @Override
571 public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
572 throws IOException {
573 postSplit.countDown();
574 }
575
576 @Override
577 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
578 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
579 preSplitBeforePONR.countDown();
580 }
581
582 @Override
583 public void start(CoprocessorEnvironment e) throws IOException {
584 postSplit = new CountDownLatch(1);
585 preSplitBeforePONR = new CountDownLatch(1);
586 }
587 }
588
589 @Test
590 public void testStatePreserve() throws Exception {
591 final String nsp1 = prefix + "_testStatePreserve";
592 NamespaceDescriptor nspDesc =
593 NamespaceDescriptor.create(nsp1)
594 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
595 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
596 ADMIN.createNamespace(nspDesc);
597 TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
598 TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
599 TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
600 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
601 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
602 tableDescOne.addFamily(fam1);
603 HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
604 tableDescTwo.addFamily(fam1);
605 HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
606 tableDescThree.addFamily(fam1);
607 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
608 ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
609 ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
610 ADMIN.disableTable(tableThree);
611 deleteTable(tableThree);
612
613 UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
614 @Override
615 public boolean evaluate() throws Exception {
616 return (getNamespaceState(nsp1).getTables().size() == 2);
617 }
618 });
619 NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
620 restartMaster();
621 NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
622 assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
623 .getTables().size(), after.getTables().size());
624 }
625
626 private static void waitForQuotaEnabled() throws Exception {
627 UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
628 @Override
629 public boolean evaluate() throws Exception {
630 HMaster master = UTIL.getHBaseCluster().getMaster();
631 if (master == null) {
632 return false;
633 }
634 MasterQuotaManager quotaManager = master.getMasterQuotaManager();
635 return quotaManager != null && quotaManager.isQuotaEnabled();
636 }
637 });
638 }
639
640 private void restartMaster() throws Exception {
641 UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
642 UTIL.getHBaseCluster().waitOnMaster(0);
643 UTIL.getHBaseCluster().startMaster();
644 waitForQuotaEnabled();
645 }
646
647 private NamespaceAuditor getQuotaManager() {
648 return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
649 }
650
651 public static class MasterSyncObserver extends BaseMasterObserver {
652 volatile CountDownLatch tableDeletionLatch;
653 static boolean throwExceptionInPreCreateTableHandler;
654
655 @Override
656 public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
657 TableName tableName) throws IOException {
658 tableDeletionLatch = new CountDownLatch(1);
659 }
660
661 @Override
662 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
663 TableName tableName) throws IOException {
664 tableDeletionLatch.countDown();
665 }
666
667 @Override
668 public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
669 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
670 if (throwExceptionInPreCreateTableHandler) {
671 throw new IOException("Throw exception as it is demanded.");
672 }
673 }
674 }
675
676 private void deleteTable(final TableName tableName) throws Exception {
677
678
679 MasterSyncObserver observer =
680 (MasterSyncObserver) UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
681 .findCoprocessor(MasterSyncObserver.class.getName());
682 ADMIN.deleteTable(tableName);
683 observer.tableDeletionLatch.await();
684 }
685
686 @Test(expected = QuotaExceededException.class)
687 public void testExceedTableQuotaInNamespace() throws Exception {
688 String nsp = prefix + "_testExceedTableQuotaInNamespace";
689 NamespaceDescriptor nspDesc =
690 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
691 .build();
692 ADMIN.createNamespace(nspDesc);
693 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
694 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
695 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
696 HTableDescriptor tableDescOne =
697 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
698 tableDescOne.addFamily(fam1);
699 HTableDescriptor tableDescTwo =
700 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
701 tableDescTwo.addFamily(fam1);
702 ADMIN.createTable(tableDescOne);
703 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
704 }
705
706 @Test(expected = QuotaExceededException.class)
707 public void testCloneSnapshotQuotaExceed() throws Exception {
708 String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
709 NamespaceDescriptor nspDesc =
710 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
711 .build();
712 ADMIN.createNamespace(nspDesc);
713 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
714 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
715 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
716 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
717 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
718 tableDescOne.addFamily(fam1);
719 ADMIN.createTable(tableDescOne);
720 String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
721 ADMIN.snapshot(snapshot, tableName);
722 ADMIN.cloneSnapshot(snapshot, cloneTableName);
723 ADMIN.deleteSnapshot(snapshot);
724 }
725
726 @Test
727 public void testCloneSnapshot() throws Exception {
728 String nsp = prefix + "_testCloneSnapshot";
729 NamespaceDescriptor nspDesc =
730 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
731 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
732 ADMIN.createNamespace(nspDesc);
733 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
734 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
735 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
736
737 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
738 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
739 tableDescOne.addFamily(fam1);
740
741 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
742 String snapshot = "snapshot_testCloneSnapshot";
743 ADMIN.snapshot(snapshot, tableName);
744 ADMIN.cloneSnapshot(snapshot, cloneTableName);
745
746 int tableLength;
747 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
748 tableLength = locator.getStartKeys().length;
749 }
750 assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
751
752 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
753 tableLength = locator.getStartKeys().length;
754 }
755 assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
756
757 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
758 assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
759 assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
760
761 ADMIN.deleteSnapshot(snapshot);
762 }
763
764 @Test
765 public void testRestoreSnapshot() throws Exception {
766 String nsp = prefix + "_testRestoreSnapshot";
767 NamespaceDescriptor nspDesc =
768 NamespaceDescriptor.create(nsp)
769 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
770 ADMIN.createNamespace(nspDesc);
771 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
772 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
773 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
774 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
775 tableDescOne.addFamily(fam1);
776 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
777
778 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
779 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
780
781 String snapshot = "snapshot_testRestoreSnapshot";
782 ADMIN.snapshot(snapshot, tableName1);
783
784 List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
785 Collections.sort(regions);
786
787 ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
788 Thread.sleep(2000);
789 assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
790
791 ADMIN.disableTable(tableName1);
792 ADMIN.restoreSnapshot(snapshot);
793
794 assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
795
796 ADMIN.enableTable(tableName1);
797 ADMIN.deleteSnapshot(snapshot);
798 }
799
800 @Test
801 public void testRestoreSnapshotQuotaExceed() throws Exception {
802 String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
803 NamespaceDescriptor nspDesc =
804 NamespaceDescriptor.create(nsp)
805 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
806 ADMIN.createNamespace(nspDesc);
807 NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
808 assertNotNull("Namespace descriptor found null.", ndesc);
809 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
810 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
811 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
812 tableDescOne.addFamily(fam1);
813
814 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
815
816 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
817 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
818
819 String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
820 ADMIN.snapshot(snapshot, tableName1);
821
822 List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
823 Collections.sort(regions);
824
825 ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
826 Thread.sleep(2000);
827 assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
828
829 ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2");
830 ADMIN.modifyNamespace(ndesc);
831
832 ADMIN.disableTable(tableName1);
833 try {
834 ADMIN.restoreSnapshot(snapshot);
835 fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
836 + " wraps IOException into RestoreSnapshotException");
837 } catch (RestoreSnapshotException ignore) {
838 }
839 ADMIN.enableTable(tableName1);
840 ADMIN.deleteSnapshot(snapshot);
841 }
842 }