1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import static org.junit.Assert.assertArrayEquals;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.TreeMap;
28
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FSDataOutputStream;
31 import org.apache.hadoop.fs.FileStatus;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HColumnDescriptor;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.client.Connection;
39 import org.apache.hadoop.hbase.client.ConnectionFactory;
40 import org.apache.hadoop.hbase.NamespaceDescriptor;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.TableNotFoundException;
43 import org.apache.hadoop.hbase.client.HTable;
44 import org.apache.hadoop.hbase.client.Table;
45 import org.apache.hadoop.hbase.codec.KeyValueCodecWithTags;
46 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
47 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
48 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
49 import org.apache.hadoop.hbase.io.hfile.HFile;
50 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
51 import org.apache.hadoop.hbase.regionserver.BloomType;
52 import org.apache.hadoop.hbase.security.SecureBulkLoadUtil;
53 import org.apache.hadoop.hbase.testclassification.LargeTests;
54 import org.apache.hadoop.hbase.util.Bytes;
55 import org.apache.hadoop.hbase.util.HFileTestUtil;
56 import org.junit.AfterClass;
57 import org.junit.BeforeClass;
58 import org.junit.Rule;
59 import org.junit.Test;
60 import org.junit.experimental.categories.Category;
61 import org.junit.rules.TestName;
62
63
64
65
66
67
68 @Category(LargeTests.class)
69 public class TestLoadIncrementalHFiles {
70 @Rule
71 public TestName tn = new TestName();
72
73 private static final byte[] QUALIFIER = Bytes.toBytes("myqual");
74 private static final byte[] FAMILY = Bytes.toBytes("myfam");
75 private static final String NAMESPACE = "bulkNS";
76
77 static final String EXPECTED_MSG_FOR_NON_EXISTING_FAMILY = "Unmatched family names found";
78 static final int MAX_FILES_PER_REGION_PER_FAMILY = 4;
79
80 private static final byte[][] SPLIT_KEYS = new byte[][] {
81 Bytes.toBytes("ddd"),
82 Bytes.toBytes("ppp")
83 };
84
85 static HBaseTestingUtility util = new HBaseTestingUtility();
86
87 @BeforeClass
88 public static void setUpBeforeClass() throws Exception {
89 util.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,"");
90 util.getConfiguration().setInt(
91 LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY,
92 MAX_FILES_PER_REGION_PER_FAMILY);
93
94 util.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY,
95 KeyValueCodecWithTags.class.getCanonicalName());
96
97 util.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, 3);
98
99 util.startMiniCluster();
100
101 setupNamespace();
102 }
103
104 protected static void setupNamespace() throws Exception {
105 util.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build());
106 }
107
108 @AfterClass
109 public static void tearDownAfterClass() throws Exception {
110 util.shutdownMiniCluster();
111 }
112
113
114
115
116
117 @Test(timeout = 120000)
118 public void testSimpleLoad() throws Exception {
119 runTest("testSimpleLoad", BloomType.NONE,
120 new byte[][][] {
121 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("cccc") },
122 new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") },
123 });
124 }
125
126
127
128
129
130 @Test(timeout = 120000)
131 public void testRegionCrossingLoad() throws Exception {
132 runTest("testRegionCrossingLoad", BloomType.NONE,
133 new byte[][][] {
134 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
135 new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
136 });
137 }
138
139
140
141
142 @Test(timeout = 60000)
143 public void testRegionCrossingRowBloom() throws Exception {
144 runTest("testRegionCrossingLoadRowBloom", BloomType.ROW,
145 new byte[][][] {
146 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
147 new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
148 });
149 }
150
151
152
153
154 @Test(timeout = 120000)
155 public void testRegionCrossingRowColBloom() throws Exception {
156 runTest("testRegionCrossingLoadRowColBloom", BloomType.ROWCOL,
157 new byte[][][] {
158 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
159 new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
160 });
161 }
162
163
164
165
166
167 @Test(timeout = 120000)
168 public void testSimpleHFileSplit() throws Exception {
169 runTest("testHFileSplit", BloomType.NONE,
170 new byte[][] {
171 Bytes.toBytes("aaa"), Bytes.toBytes("fff"), Bytes.toBytes("jjj"),
172 Bytes.toBytes("ppp"), Bytes.toBytes("uuu"), Bytes.toBytes("zzz"),
173 },
174 new byte[][][] {
175 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("lll") },
176 new byte[][]{ Bytes.toBytes("mmm"), Bytes.toBytes("zzz") },
177 }
178 );
179 }
180
181
182
183
184
185 @Test(timeout = 60000)
186 public void testRegionCrossingHFileSplit() throws Exception {
187 testRegionCrossingHFileSplit(BloomType.NONE);
188 }
189
190
191
192
193
194 @Test(timeout = 120000)
195 public void testRegionCrossingHFileSplitRowBloom() throws Exception {
196 testRegionCrossingHFileSplit(BloomType.ROW);
197 }
198
199
200
201
202
203 @Test(timeout = 120000)
204 public void testRegionCrossingHFileSplitRowColBloom() throws Exception {
205 testRegionCrossingHFileSplit(BloomType.ROWCOL);
206 }
207
208 @Test
209 public void testSplitALot() throws Exception {
210 runTest("testSplitALot", BloomType.NONE,
211 new byte[][] {
212 Bytes.toBytes("aaaa"), Bytes.toBytes("bbb"),
213 Bytes.toBytes("ccc"), Bytes.toBytes("ddd"),
214 Bytes.toBytes("eee"), Bytes.toBytes("fff"),
215 Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
216 Bytes.toBytes("iii"), Bytes.toBytes("lll"),
217 Bytes.toBytes("mmm"), Bytes.toBytes("nnn"),
218 Bytes.toBytes("ooo"), Bytes.toBytes("ppp"),
219 Bytes.toBytes("qqq"), Bytes.toBytes("rrr"),
220 Bytes.toBytes("sss"), Bytes.toBytes("ttt"),
221 Bytes.toBytes("uuu"), Bytes.toBytes("vvv"),
222 Bytes.toBytes("zzz"),
223 },
224 new byte[][][] {
225 new byte[][] { Bytes.toBytes("aaaa"), Bytes.toBytes("zzz") },
226 }
227 );
228 }
229
230 private void testRegionCrossingHFileSplit(BloomType bloomType) throws Exception {
231 runTest("testHFileSplit" + bloomType + "Bloom", bloomType,
232 new byte[][] {
233 Bytes.toBytes("aaa"), Bytes.toBytes("fff"), Bytes.toBytes("jjj"),
234 Bytes.toBytes("ppp"), Bytes.toBytes("uuu"), Bytes.toBytes("zzz"),
235 },
236 new byte[][][] {
237 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
238 new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
239 }
240 );
241 }
242
243 private HTableDescriptor buildHTD(TableName tableName, BloomType bloomType) {
244 HTableDescriptor htd = new HTableDescriptor(tableName);
245 HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
246 familyDesc.setBloomFilterType(bloomType);
247 htd.addFamily(familyDesc);
248 return htd;
249 }
250
251 private void runTest(String testName, BloomType bloomType,
252 byte[][][] hfileRanges) throws Exception {
253 runTest(testName, bloomType, null, hfileRanges);
254 }
255
256 private void runTest(String testName, BloomType bloomType,
257 byte[][] tableSplitKeys, byte[][][] hfileRanges) throws Exception {
258 final byte[] TABLE_NAME = Bytes.toBytes("mytable_"+testName);
259 final boolean preCreateTable = tableSplitKeys != null;
260
261
262 final TableName TABLE_WITHOUT_NS = TableName.valueOf(TABLE_NAME);
263 runTest(testName, TABLE_WITHOUT_NS, bloomType, preCreateTable, tableSplitKeys, hfileRanges);
264
265
266 final TableName TABLE_WITH_NS = TableName.valueOf(Bytes.toBytes(NAMESPACE), TABLE_NAME);
267 runTest(testName, TABLE_WITH_NS, bloomType, preCreateTable, tableSplitKeys, hfileRanges);
268 }
269
270 private void runTest(String testName, TableName tableName, BloomType bloomType,
271 boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges) throws Exception {
272 HTableDescriptor htd = buildHTD(tableName, bloomType);
273 runTest(testName, htd, bloomType, preCreateTable, tableSplitKeys, hfileRanges);
274 }
275
276 private void runTest(String testName, HTableDescriptor htd, BloomType bloomType,
277 boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges) throws Exception {
278
279 for (boolean managed : new boolean[] { true, false }) {
280 Path dir = util.getDataTestDirOnTestFS(testName);
281 FileSystem fs = util.getTestFileSystem();
282 dir = dir.makeQualified(fs);
283 Path familyDir = new Path(dir, Bytes.toString(FAMILY));
284
285 int hfileIdx = 0;
286 for (byte[][] range : hfileRanges) {
287 byte[] from = range[0];
288 byte[] to = range[1];
289 HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
290 + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000);
291 }
292 int expectedRows = hfileIdx * 1000;
293
294 if (preCreateTable) {
295 util.getHBaseAdmin().createTable(htd, tableSplitKeys);
296 }
297
298 final TableName tableName = htd.getTableName();
299 if (!util.getHBaseAdmin().tableExists(tableName)) {
300 util.getHBaseAdmin().createTable(htd);
301 }
302 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
303
304 if (managed) {
305 try (HTable table = new HTable(util.getConfiguration(), tableName)) {
306 loader.doBulkLoad(dir, table);
307 assertEquals(expectedRows, util.countRows(table));
308 }
309 } else {
310 try (Connection conn = ConnectionFactory.createConnection(util.getConfiguration());
311 HTable table = (HTable) conn.getTable(tableName)) {
312 loader.doBulkLoad(dir, table);
313 }
314 }
315
316
317 Path stagingBasePath = SecureBulkLoadUtil.getBaseStagingDir(util.getConfiguration());
318 if (fs.exists(stagingBasePath)) {
319 FileStatus[] files = fs.listStatus(stagingBasePath);
320 for (FileStatus file : files) {
321 assertTrue("Folder=" + file.getPath() + " is not cleaned up.",
322 file.getPath().getName() != "DONOTERASE");
323 }
324 }
325
326 util.deleteTable(tableName);
327 }
328 }
329
330
331
332
333
334
335
336 @Test(timeout = 60000)
337 public void htestTagsSurviveBulkLoadSplit() throws Exception {
338 Path dir = util.getDataTestDirOnTestFS(tn.getMethodName());
339 FileSystem fs = util.getTestFileSystem();
340 dir = dir.makeQualified(fs);
341 Path familyDir = new Path(dir, Bytes.toString(FAMILY));
342
343 byte [][] tableSplitKeys = new byte[][] {
344 Bytes.toBytes("aaa"), Bytes.toBytes("fff"), Bytes.toBytes("jjj"),
345 Bytes.toBytes("ppp"), Bytes.toBytes("uuu"), Bytes.toBytes("zzz"),
346 };
347
348
349 byte[] from = Bytes.toBytes("ddd");
350 byte[] to = Bytes.toBytes("ooo");
351 HFileTestUtil.createHFileWithTags(util.getConfiguration(), fs,
352 new Path(familyDir, tn.getMethodName()+"_hfile"),
353 FAMILY, QUALIFIER, from, to, 1000);
354 int expectedRows = 1000;
355
356 TableName tableName = TableName.valueOf(tn.getMethodName());
357 HTableDescriptor htd = buildHTD(tableName, BloomType.NONE);
358 util.getHBaseAdmin().createTable(htd, tableSplitKeys);
359
360 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
361 String [] args= {dir.toString(), tableName.toString()};
362 loader.run(args);
363
364 Table table = util.getConnection().getTable(tableName);
365 try {
366 assertEquals(expectedRows, util.countRows(table));
367 HFileTestUtil.verifyTags(table);
368 } finally {
369 table.close();
370 }
371
372 util.deleteTable(tableName);
373 }
374
375
376
377
378 @Test(timeout = 60000)
379 public void testNonexistentColumnFamilyLoad() throws Exception {
380 String testName = "testNonexistentColumnFamilyLoad";
381 byte[][][] hFileRanges = new byte[][][] {
382 new byte[][]{ Bytes.toBytes("aaa"), Bytes.toBytes("ccc") },
383 new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") },
384 };
385
386 final byte[] TABLE = Bytes.toBytes("mytable_"+testName);
387 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
388
389
390 HColumnDescriptor family =
391 new HColumnDescriptor(Bytes.toBytes(new String(FAMILY).toUpperCase()));
392 htd.addFamily(family);
393
394 try {
395 runTest(testName, htd, BloomType.NONE, true, SPLIT_KEYS, hFileRanges);
396 assertTrue("Loading into table with non-existent family should have failed", false);
397 } catch (Exception e) {
398 assertTrue("IOException expected", e instanceof IOException);
399
400 String errMsg = e.getMessage();
401 assertTrue("Incorrect exception message, expected message: ["
402 + EXPECTED_MSG_FOR_NON_EXISTING_FAMILY + "], current message: [" + errMsg + "]",
403 errMsg.contains(EXPECTED_MSG_FOR_NON_EXISTING_FAMILY));
404 }
405 }
406
407 @Test(timeout = 120000)
408 public void testNonHfileFolderWithUnmatchedFamilyName() throws Exception {
409 testNonHfileFolder("testNonHfileFolderWithUnmatchedFamilyName", true);
410 }
411
412 @Test(timeout = 120000)
413 public void testNonHfileFolder() throws Exception {
414 testNonHfileFolder("testNonHfileFolder", false);
415 }
416
417
418
419
420
421
422 private void testNonHfileFolder(String tableName, boolean preCreateTable) throws Exception {
423 Path dir = util.getDataTestDirOnTestFS(tableName);
424 FileSystem fs = util.getTestFileSystem();
425 dir = dir.makeQualified(fs);
426
427 Path familyDir = new Path(dir, Bytes.toString(FAMILY));
428 HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_0"),
429 FAMILY, QUALIFIER, Bytes.toBytes("begin"), Bytes.toBytes("end"), 500);
430 createRandomDataFile(fs, new Path(familyDir, "012356789"), 16 * 1024);
431
432 final String NON_FAMILY_FOLDER = "_logs";
433 Path nonFamilyDir = new Path(dir, NON_FAMILY_FOLDER);
434 fs.mkdirs(nonFamilyDir);
435 fs.mkdirs(new Path(nonFamilyDir, "non-file"));
436 createRandomDataFile(fs, new Path(nonFamilyDir, "012356789"), 16 * 1024);
437
438 Table table = null;
439 try {
440 if (preCreateTable) {
441 table = util.createTable(TableName.valueOf(tableName), FAMILY);
442 } else {
443 table = util.getConnection().getTable(TableName.valueOf(tableName));
444 }
445
446 final String[] args = {dir.toString(), tableName};
447 new LoadIncrementalHFiles(util.getConfiguration()).run(args);
448 assertEquals(500, util.countRows(table));
449 } finally {
450 if (table != null) {
451 table.close();
452 }
453 fs.delete(dir, true);
454 }
455 }
456
457 private static void createRandomDataFile(FileSystem fs, Path path, int size)
458 throws IOException {
459 FSDataOutputStream stream = fs.create(path);
460 try {
461 byte[] data = new byte[1024];
462 for (int i = 0; i < data.length; ++i) {
463 data[i] = (byte)(i & 0xff);
464 }
465 while (size >= data.length) {
466 stream.write(data, 0, data.length);
467 size -= data.length;
468 }
469 if (size > 0) {
470 stream.write(data, 0, size);
471 }
472 } finally {
473 stream.close();
474 }
475 }
476
477 @Test(timeout = 120000)
478 public void testSplitStoreFile() throws IOException {
479 Path dir = util.getDataTestDirOnTestFS("testSplitHFile");
480 FileSystem fs = util.getTestFileSystem();
481 Path testIn = new Path(dir, "testhfile");
482 HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
483 HFileTestUtil.createHFile(util.getConfiguration(), fs, testIn, FAMILY, QUALIFIER,
484 Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000);
485
486 Path bottomOut = new Path(dir, "bottom.out");
487 Path topOut = new Path(dir, "top.out");
488
489 LoadIncrementalHFiles.splitStoreFile(
490 util.getConfiguration(), testIn,
491 familyDesc, Bytes.toBytes("ggg"),
492 bottomOut,
493 topOut);
494
495 int rowCount = verifyHFile(bottomOut);
496 rowCount += verifyHFile(topOut);
497 assertEquals(1000, rowCount);
498 }
499
500 @Test
501 public void testSplitStoreFileWithNoneToNone() throws IOException {
502 testSplitStoreFileWithDifferentEncoding(DataBlockEncoding.NONE, DataBlockEncoding.NONE);
503 }
504
505 @Test
506 public void testSplitStoreFileWithEncodedToEncoded() throws IOException {
507 testSplitStoreFileWithDifferentEncoding(DataBlockEncoding.DIFF, DataBlockEncoding.DIFF);
508 }
509
510 @Test
511 public void testSplitStoreFileWithEncodedToNone() throws IOException {
512 testSplitStoreFileWithDifferentEncoding(DataBlockEncoding.DIFF, DataBlockEncoding.NONE);
513 }
514
515 @Test
516 public void testSplitStoreFileWithNoneToEncoded() throws IOException {
517 testSplitStoreFileWithDifferentEncoding(DataBlockEncoding.NONE, DataBlockEncoding.DIFF);
518 }
519
520 private void testSplitStoreFileWithDifferentEncoding(DataBlockEncoding bulkloadEncoding,
521 DataBlockEncoding cfEncoding) throws IOException {
522 Path dir = util.getDataTestDirOnTestFS("testSplitHFileWithDifferentEncoding");
523 FileSystem fs = util.getTestFileSystem();
524 Path testIn = new Path(dir, "testhfile");
525 HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
526 familyDesc.setDataBlockEncoding(cfEncoding);
527 HFileTestUtil.createHFileWithDataBlockEncoding(
528 util.getConfiguration(), fs, testIn, bulkloadEncoding,
529 FAMILY, QUALIFIER, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000);
530
531 Path bottomOut = new Path(dir, "bottom.out");
532 Path topOut = new Path(dir, "top.out");
533
534 LoadIncrementalHFiles.splitStoreFile(
535 util.getConfiguration(), testIn,
536 familyDesc, Bytes.toBytes("ggg"),
537 bottomOut,
538 topOut);
539
540 int rowCount = verifyHFile(bottomOut);
541 rowCount += verifyHFile(topOut);
542 assertEquals(1000, rowCount);
543 }
544
545 private int verifyHFile(Path p) throws IOException {
546 Configuration conf = util.getConfiguration();
547 HFile.Reader reader = HFile.createReader(
548 p.getFileSystem(conf), p, new CacheConfig(conf), conf);
549 reader.loadFileInfo();
550 HFileScanner scanner = reader.getScanner(false, false);
551 scanner.seekTo();
552 int count = 0;
553 do {
554 count++;
555 } while (scanner.next());
556 assertTrue(count > 0);
557 reader.close();
558 return count;
559 }
560
561 private void addStartEndKeysForTest(TreeMap<byte[], Integer> map, byte[] first, byte[] last) {
562 Integer value = map.containsKey(first)?map.get(first):0;
563 map.put(first, value+1);
564
565 value = map.containsKey(last)?map.get(last):0;
566 map.put(last, value-1);
567 }
568
569 @Test(timeout = 120000)
570 public void testInferBoundaries() {
571 TreeMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
572
573
574
575
576
577
578
579
580
581
582
583 String first;
584 String last;
585
586 first = "a"; last = "e";
587 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
588
589 first = "r"; last = "s";
590 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
591
592 first = "o"; last = "p";
593 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
594
595 first = "g"; last = "k";
596 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
597
598 first = "v"; last = "x";
599 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
600
601 first = "c"; last = "i";
602 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
603
604 first = "m"; last = "q";
605 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
606
607 first = "s"; last = "t";
608 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
609
610 first = "u"; last = "w";
611 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
612
613 byte[][] keysArray = LoadIncrementalHFiles.inferBoundaries(map);
614 byte[][] compare = new byte[3][];
615 compare[0] = "m".getBytes();
616 compare[1] = "r".getBytes();
617 compare[2] = "u".getBytes();
618
619 assertEquals(keysArray.length, 3);
620
621 for (int row = 0; row<keysArray.length; row++){
622 assertArrayEquals(keysArray[row], compare[row]);
623 }
624 }
625
626 @Test(timeout = 60000)
627 public void testLoadTooMayHFiles() throws Exception {
628 Path dir = util.getDataTestDirOnTestFS("testLoadTooMayHFiles");
629 FileSystem fs = util.getTestFileSystem();
630 dir = dir.makeQualified(fs);
631 Path familyDir = new Path(dir, Bytes.toString(FAMILY));
632
633 byte[] from = Bytes.toBytes("begin");
634 byte[] to = Bytes.toBytes("end");
635 for (int i = 0; i <= MAX_FILES_PER_REGION_PER_FAMILY; i++) {
636 HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
637 + i), FAMILY, QUALIFIER, from, to, 1000);
638 }
639
640 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
641 String [] args= {dir.toString(), "mytable_testLoadTooMayHFiles"};
642 try {
643 loader.run(args);
644 fail("Bulk loading too many files should fail");
645 } catch (IOException ie) {
646 assertTrue(ie.getMessage().contains("Trying to load more than "
647 + MAX_FILES_PER_REGION_PER_FAMILY + " hfiles"));
648 }
649 }
650
651 @Test(expected = TableNotFoundException.class)
652 public void testWithoutAnExistingTableAndCreateTableSetToNo() throws Exception {
653 Configuration conf = util.getConfiguration();
654 conf.set(LoadIncrementalHFiles.CREATE_TABLE_CONF_KEY, "no");
655 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
656 String[] args = { "directory", "nonExistingTable" };
657 loader.run(args);
658 }
659
660 @Test(timeout = 120000)
661 public void testTableWithCFNameStartWithUnderScore() throws Exception {
662 Path dir = util.getDataTestDirOnTestFS("cfNameStartWithUnderScore");
663 FileSystem fs = util.getTestFileSystem();
664 dir = dir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
665 String family = "_cf";
666 Path familyDir = new Path(dir, family);
667
668 byte[] from = Bytes.toBytes("begin");
669 byte[] to = Bytes.toBytes("end");
670 Configuration conf = util.getConfiguration();
671 String tableName = "mytable_cfNameStartWithUnderScore";
672 Table table = util.createTable(TableName.valueOf(tableName), family);
673 HFileTestUtil.createHFile(conf, fs, new Path(familyDir, "hfile"), Bytes.toBytes(family),
674 QUALIFIER, from, to, 1000);
675
676 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
677 String[] args = { dir.toString(), tableName };
678 try {
679 loader.run(args);
680 assertEquals(1000, util.countRows(table));
681 } finally {
682 if (null != table) {
683 table.close();
684 }
685 }
686 }
687 }
688