1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.HashSet;
26 import java.util.Set;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.HColumnDescriptor;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.master.MasterFileSystem;
37 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
38 import org.apache.hadoop.hbase.mob.MobConstants;
39 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
40 import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
41 import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils;
42 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
43 import org.apache.hadoop.hbase.testclassification.LargeTests;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.FSUtils;
46 import org.junit.After;
47 import org.junit.AfterClass;
48 import org.junit.Before;
49 import org.junit.BeforeClass;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52
53
54
55
56 @Category(LargeTests.class)
57 public class TestMobRestoreSnapshotFromClient {
58 final Log LOG = LogFactory.getLog(getClass());
59
60 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61
62 private final byte[] FAMILY = Bytes.toBytes("cf");
63
64 private byte[] emptySnapshot;
65 private byte[] snapshotName0;
66 private byte[] snapshotName1;
67 private byte[] snapshotName2;
68 private int snapshot0Rows;
69 private int snapshot1Rows;
70 private TableName tableName;
71 private Admin admin;
72
73 @BeforeClass
74 public static void setUpBeforeClass() throws Exception {
75 TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
76 TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
77 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
78 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
79 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
80 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
81 TEST_UTIL.getConfiguration().setBoolean(
82 "hbase.master.enabletable.roundrobin", true);
83 TEST_UTIL.getConfiguration().setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0);
84 TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
85 TEST_UTIL.startMiniCluster(3);
86 }
87
88 @AfterClass
89 public static void tearDownAfterClass() throws Exception {
90 TEST_UTIL.shutdownMiniCluster();
91 }
92
93
94
95
96
97
98 @Before
99 public void setup() throws Exception {
100 this.admin = TEST_UTIL.getHBaseAdmin();
101
102 long tid = System.currentTimeMillis();
103 tableName =
104 TableName.valueOf("testtb-" + tid);
105 emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
106 snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
107 snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
108 snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
109
110
111 MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
112
113 admin.disableTable(tableName);
114
115
116 admin.snapshot(emptySnapshot, tableName);
117
118 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
119
120 admin.enableTable(tableName);
121 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
122 snapshot0Rows = MobSnapshotTestingUtils.countMobRows(table);
123 admin.disableTable(tableName);
124
125
126 admin.snapshot(snapshotName0, tableName);
127
128
129 admin.enableTable(tableName);
130 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
131 snapshot1Rows = MobSnapshotTestingUtils.countMobRows(table);
132 table.close();
133 }
134
135 @After
136 public void tearDown() throws Exception {
137 TEST_UTIL.deleteTable(tableName);
138 SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
139 SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
140 }
141
142 @Test
143 public void testRestoreSnapshot() throws IOException {
144 MobSnapshotTestingUtils.verifyMobRowCount(TEST_UTIL, tableName, snapshot1Rows);
145 admin.disableTable(tableName);
146 admin.snapshot(snapshotName1, tableName);
147
148 admin.restoreSnapshot(snapshotName0);
149 admin.enableTable(tableName);
150 MobSnapshotTestingUtils.verifyMobRowCount(TEST_UTIL, tableName, snapshot0Rows);
151 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
152
153
154 admin.disableTable(tableName);
155 admin.restoreSnapshot(emptySnapshot);
156 admin.enableTable(tableName);
157 MobSnapshotTestingUtils.verifyMobRowCount(TEST_UTIL, tableName, 0);
158 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
159
160
161 admin.disableTable(tableName);
162 admin.restoreSnapshot(snapshotName1);
163 admin.enableTable(tableName);
164 MobSnapshotTestingUtils.verifyMobRowCount(TEST_UTIL, tableName, snapshot1Rows);
165 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
166
167
168 TEST_UTIL.deleteTable(tableName);
169 admin.restoreSnapshot(snapshotName1);
170 MobSnapshotTestingUtils.verifyMobRowCount(TEST_UTIL, tableName, snapshot1Rows);
171 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
172 }
173
174 protected int getNumReplicas() {
175 return 1;
176 }
177
178 @Test
179 public void testRestoreSchemaChange() throws Exception {
180 byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
181
182 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
183
184
185 admin.disableTable(tableName);
186 HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY2);
187 hcd.setMobEnabled(true);
188 hcd.setMobThreshold(3L);
189 admin.addColumn(tableName, hcd);
190 admin.enableTable(tableName);
191 assertEquals(2, table.getTableDescriptor().getFamilies().size());
192 HTableDescriptor htd = admin.getTableDescriptor(tableName);
193 assertEquals(2, htd.getFamilies().size());
194 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, TEST_FAMILY2);
195 long snapshot2Rows = snapshot1Rows + 500;
196 assertEquals(snapshot2Rows, MobSnapshotTestingUtils.countMobRows(table));
197 assertEquals(500, MobSnapshotTestingUtils.countMobRows(table, TEST_FAMILY2));
198 Set<String> fsFamilies = getFamiliesFromFS(tableName);
199 assertEquals(2, fsFamilies.size());
200
201
202 admin.disableTable(tableName);
203 admin.snapshot(snapshotName2, tableName);
204
205
206 admin.restoreSnapshot(snapshotName0);
207 admin.enableTable(tableName);
208 assertEquals(1, table.getTableDescriptor().getFamilies().size());
209 try {
210 MobSnapshotTestingUtils.countMobRows(table, TEST_FAMILY2);
211 fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
212 } catch (NoSuchColumnFamilyException e) {
213
214 }
215 assertEquals(snapshot0Rows, MobSnapshotTestingUtils.countMobRows(table));
216 htd = admin.getTableDescriptor(tableName);
217 assertEquals(1, htd.getFamilies().size());
218 fsFamilies = getFamiliesFromFS(tableName);
219 assertEquals(1, fsFamilies.size());
220
221
222 admin.disableTable(tableName);
223 admin.restoreSnapshot(snapshotName2);
224 admin.enableTable(tableName);
225 htd = admin.getTableDescriptor(tableName);
226 assertEquals(2, htd.getFamilies().size());
227 assertEquals(2, table.getTableDescriptor().getFamilies().size());
228 assertEquals(500, MobSnapshotTestingUtils.countMobRows(table, TEST_FAMILY2));
229 assertEquals(snapshot2Rows, MobSnapshotTestingUtils.countMobRows(table));
230 fsFamilies = getFamiliesFromFS(tableName);
231 assertEquals(2, fsFamilies.size());
232 table.close();
233 }
234
235 @Test
236 public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
237 TableName clonedTableName =
238 TableName.valueOf("clonedtb-" + System.currentTimeMillis());
239 admin.cloneSnapshot(snapshotName0, clonedTableName);
240 MobSnapshotTestingUtils.verifyMobRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
241 SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
242 admin.disableTable(clonedTableName);
243 admin.snapshot(snapshotName2, clonedTableName);
244 TEST_UTIL.deleteTable(clonedTableName);
245 waitCleanerRun();
246
247 admin.cloneSnapshot(snapshotName2, clonedTableName);
248 MobSnapshotTestingUtils.verifyMobRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
249 SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
250 TEST_UTIL.deleteTable(clonedTableName);
251 }
252
253 @Test
254 public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
255 TEST_UTIL.deleteTable(tableName);
256 waitCleanerRun();
257
258 admin.cloneSnapshot(snapshotName0, tableName);
259 MobSnapshotTestingUtils.verifyMobRowCount(TEST_UTIL, tableName, snapshot0Rows);
260 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
261 waitCleanerRun();
262
263 admin.disableTable(tableName);
264 admin.restoreSnapshot(snapshotName0);
265 admin.enableTable(tableName);
266 MobSnapshotTestingUtils.verifyMobRowCount(TEST_UTIL, tableName, snapshot0Rows);
267 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
268 }
269
270 @Test
271 public void testCorruptedSnapshot() throws IOException, InterruptedException {
272 SnapshotTestingUtils.corruptSnapshot(TEST_UTIL, Bytes.toString(snapshotName0));
273 TableName cloneName = TableName.valueOf("corruptedClone-" + System.currentTimeMillis());
274 try {
275 admin.cloneSnapshot(snapshotName0, cloneName);
276 fail("Expected CorruptedSnapshotException, got succeeded cloneSnapshot()");
277 } catch (CorruptedSnapshotException e) {
278
279
280 assertFalse(admin.tableExists(cloneName));
281 } catch (Exception e) {
282 fail("Expected CorruptedSnapshotException got: " + e);
283 }
284 }
285
286
287
288
289 private void waitCleanerRun() throws InterruptedException {
290 TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
291 }
292
293 private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
294 MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
295 Set<String> families = new HashSet<String>();
296 Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
297 for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
298 for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
299 families.add(familyDir.getName());
300 }
301 }
302 return families;
303 }
304 }