View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.hbase.DoNotRetryIOException;
29  import org.apache.hadoop.hbase.HBaseTestingUtility;
30  import org.apache.hadoop.hbase.HColumnDescriptor;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.ProcedureInfo;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
37  import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
38  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ModifyTableState;
39  import org.apache.hadoop.hbase.testclassification.MediumTests;
40  import org.junit.After;
41  import org.junit.AfterClass;
42  import org.junit.Before;
43  import org.junit.BeforeClass;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  
47  @Category(MediumTests.class)
48  public class TestModifyTableProcedure {
49    private static final Log LOG = LogFactory.getLog(TestModifyTableProcedure.class);
50  
51    protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
52  
53    private static long nonceGroup = HConstants.NO_NONCE;
54    private static long nonce = HConstants.NO_NONCE;
55  
56    private static void setupConf(Configuration conf) {
57      conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
58    }
59  
60    @BeforeClass
61    public static void setupCluster() throws Exception {
62      setupConf(UTIL.getConfiguration());
63      UTIL.startMiniCluster(1);
64    }
65  
66    @AfterClass
67    public static void cleanupTest() throws Exception {
68      try {
69        UTIL.shutdownMiniCluster();
70      } catch (Exception e) {
71        LOG.warn("failure shutting down cluster", e);
72      }
73    }
74  
75    @Before
76    public void setup() throws Exception {
77      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
78      nonceGroup =
79          MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
80      nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
81    }
82  
83    @After
84    public void tearDown() throws Exception {
85      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
86      for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) {
87        LOG.info("Tear down, remove table=" + htd.getTableName());
88        UTIL.deleteTable(htd.getTableName());
89      }
90    }
91  
92    @Test(timeout=60000)
93    public void testModifyTable() throws Exception {
94      final TableName tableName = TableName.valueOf("testModifyTable");
95      final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
96  
97      MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf");
98      UTIL.getHBaseAdmin().disableTable(tableName);
99  
100     // Modify the table descriptor
101     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
102 
103     // Test 1: Modify 1 property
104     long newMaxFileSize = htd.getMaxFileSize() * 2;
105     htd.setMaxFileSize(newMaxFileSize);
106     htd.setRegionReplication(3);
107 
108     long procId1 = ProcedureTestingUtility.submitAndWait(
109         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
110     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
111 
112     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
113     assertEquals(newMaxFileSize, currentHtd.getMaxFileSize());
114 
115     // Test 2: Modify multiple properties
116     boolean newReadOnlyOption = htd.isReadOnly() ? false : true;
117     long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2;
118     htd.setReadOnly(newReadOnlyOption);
119     htd.setMemStoreFlushSize(newMemStoreFlushSize);
120 
121     long procId2 = ProcedureTestingUtility.submitAndWait(
122         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
123     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
124 
125     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
126     assertEquals(newReadOnlyOption, currentHtd.isReadOnly());
127     assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize());
128   }
129 
130   @Test(timeout = 60000)
131   public void testModifyTableAddCF() throws Exception {
132     final TableName tableName = TableName.valueOf("testModifyTableAddCF");
133     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
134 
135     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1");
136     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
137     assertEquals(1, currentHtd.getFamiliesKeys().size());
138 
139     // Test 1: Modify the table descriptor online
140     String cf2 = "cf2";
141     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
142     htd.addFamily(new HColumnDescriptor(cf2));
143 
144     long procId = ProcedureTestingUtility.submitAndWait(
145         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
146     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
147 
148     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
149     assertEquals(2, currentHtd.getFamiliesKeys().size());
150     assertTrue(currentHtd.hasFamily(cf2.getBytes()));
151 
152     // Test 2: Modify the table descriptor offline
153     UTIL.getHBaseAdmin().disableTable(tableName);
154     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
155     String cf3 = "cf3";
156     HTableDescriptor htd2 =
157         new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
158     htd2.addFamily(new HColumnDescriptor(cf3));
159 
160     long procId2 =
161         ProcedureTestingUtility.submitAndWait(procExec,
162           new ModifyTableProcedure(procExec.getEnvironment(), htd2));
163     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
164 
165     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
166     assertTrue(currentHtd.hasFamily(cf3.getBytes()));
167     assertEquals(3, currentHtd.getFamiliesKeys().size());
168   }
169 
170   @Test(timeout = 60000)
171   public void testModifyTableDeleteCF() throws Exception {
172     final TableName tableName = TableName.valueOf("testModifyTableDeleteCF");
173     final String cf1 = "cf1";
174     final String cf2 = "cf2";
175     final String cf3 = "cf3";
176     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
177 
178     MasterProcedureTestingUtility.createTable(procExec, tableName, null, cf1, cf2, cf3);
179     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
180     assertEquals(3, currentHtd.getFamiliesKeys().size());
181 
182     // Test 1: Modify the table descriptor
183     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
184     htd.removeFamily(cf2.getBytes());
185 
186     long procId = ProcedureTestingUtility.submitAndWait(
187         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
188     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
189 
190     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
191     assertEquals(2, currentHtd.getFamiliesKeys().size());
192     assertFalse(currentHtd.hasFamily(cf2.getBytes()));
193 
194     // Test 2: Modify the table descriptor offline
195     UTIL.getHBaseAdmin().disableTable(tableName);
196     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
197 
198     HTableDescriptor htd2 =
199         new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
200     htd2.removeFamily(cf3.getBytes());
201     // Disable Sanity check
202     htd2.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());
203 
204     long procId2 =
205         ProcedureTestingUtility.submitAndWait(procExec,
206           new ModifyTableProcedure(procExec.getEnvironment(), htd2));
207     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
208 
209     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
210     assertEquals(1, currentHtd.getFamiliesKeys().size());
211     assertFalse(currentHtd.hasFamily(cf3.getBytes()));
212 
213     //Removing the last family will fail
214     HTableDescriptor htd3 =
215         new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
216     htd3.removeFamily(cf1.getBytes());
217     long procId3 =
218         ProcedureTestingUtility.submitAndWait(procExec,
219             new ModifyTableProcedure(procExec.getEnvironment(), htd3));
220     final ProcedureInfo result = procExec.getResult(procId3);
221     assertEquals(true, result.isFailed());
222     Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
223     assertTrue("expected DoNotRetryIOException, got " + cause,
224         cause instanceof DoNotRetryIOException);
225     assertEquals(1, currentHtd.getFamiliesKeys().size());
226     assertTrue(currentHtd.hasFamily(cf1.getBytes()));
227   }
228 
229   @Test(timeout=60000)
230   public void testRecoveryAndDoubleExecutionOffline() throws Exception {
231     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOffline");
232     final String cf2 = "cf2";
233     final String cf3 = "cf3";
234     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
235 
236     // create the table
237     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
238       procExec, tableName, null, "cf1", cf3);
239     UTIL.getHBaseAdmin().disableTable(tableName);
240 
241     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
242     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
243 
244     // Modify multiple properties of the table.
245     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
246     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
247     htd.setCompactionEnabled(newCompactionEnableOption);
248     htd.addFamily(new HColumnDescriptor(cf2));
249     htd.removeFamily(cf3.getBytes());
250     htd.setRegionReplication(3);
251 
252     // Start the Modify procedure && kill the executor
253     long procId = procExec.submitProcedure(
254       new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
255 
256     // Restart the executor and execute the step twice
257     int numberOfSteps = ModifyTableState.values().length;
258     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
259       procExec,
260       procId,
261       numberOfSteps,
262       ModifyTableState.values());
263 
264     // Validate descriptor
265     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
266     assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
267     assertEquals(2, currentHtd.getFamiliesKeys().size());
268 
269     // cf2 should be added cf3 should be removed
270     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
271       tableName, regions, false, "cf1", cf2);
272   }
273 
274   @Test(timeout = 60000)
275   public void testRecoveryAndDoubleExecutionOnline() throws Exception {
276     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
277     final String cf2 = "cf2";
278     final String cf3 = "cf3";
279     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
280 
281     // create the table
282     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
283       procExec, tableName, null, "cf1", cf3);
284 
285     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
286 
287     // Modify multiple properties of the table.
288     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
289     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
290     htd.setCompactionEnabled(newCompactionEnableOption);
291     htd.addFamily(new HColumnDescriptor(cf2));
292     htd.removeFamily(cf3.getBytes());
293 
294     // Start the Modify procedure && kill the executor
295     long procId = procExec.submitProcedure(
296       new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
297 
298     // Restart the executor and execute the step twice
299     int numberOfSteps = ModifyTableState.values().length;
300     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
301       ModifyTableState.values());
302 
303     // Validate descriptor
304     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
305     assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
306     assertEquals(2, currentHtd.getFamiliesKeys().size());
307     assertTrue(currentHtd.hasFamily(cf2.getBytes()));
308     assertFalse(currentHtd.hasFamily(cf3.getBytes()));
309 
310     // cf2 should be added cf3 should be removed
311     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
312       tableName, regions, "cf1", cf2);
313   }
314 
315   @Test(timeout = 60000)
316   public void testRollbackAndDoubleExecutionOnline() throws Exception {
317     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
318     final String familyName = "cf2";
319     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
320 
321     // create the table
322     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
323       procExec, tableName, null, "cf1");
324 
325     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
326 
327     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
328     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
329     htd.setCompactionEnabled(newCompactionEnableOption);
330     htd.addFamily(new HColumnDescriptor(familyName));
331 
332     // Start the Modify procedure && kill the executor
333     long procId = procExec.submitProcedure(
334       new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
335 
336     // Restart the executor and rollback the step twice
337     int numberOfSteps = ModifyTableState.values().length - 4; // failing in the middle of proc
338     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
339       procExec,
340       procId,
341       numberOfSteps,
342       ModifyTableState.values());
343 
344     // cf2 should not be present
345     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
346       tableName, regions, "cf1");
347   }
348 
349   @Test(timeout = 60000)
350   public void testRollbackAndDoubleExecutionOffline() throws Exception {
351     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
352     final String familyName = "cf2";
353     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
354 
355     // create the table
356     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
357       procExec, tableName, null, "cf1");
358     UTIL.getHBaseAdmin().disableTable(tableName);
359 
360     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
361     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
362 
363     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
364     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
365     htd.setCompactionEnabled(newCompactionEnableOption);
366     htd.addFamily(new HColumnDescriptor(familyName));
367     htd.setRegionReplication(3);
368 
369     // Start the Modify procedure && kill the executor
370     long procId = procExec.submitProcedure(
371       new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
372 
373     // Restart the executor and rollback the step twice
374     int numberOfSteps = ModifyTableState.values().length - 4; // failing in the middle of proc
375     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
376       procExec,
377       procId,
378       numberOfSteps,
379       ModifyTableState.values());
380 
381     // cf2 should not be present
382     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
383       tableName, regions, "cf1");
384   }
385 
386   @Test(timeout = 60000)
387   public void testRollbackAndDoubleExecutionAfterPONR() throws Exception {
388     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecutionAfterPONR");
389     final String familyToAddName = "cf2";
390     final String familyToRemove = "cf1";
391     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
392 
393     // create the table
394     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
395       procExec, tableName, null, familyToRemove);
396     UTIL.getHBaseAdmin().disableTable(tableName);
397 
398     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
399     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
400 
401     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
402     htd.setCompactionEnabled(!htd.isCompactionEnabled());
403     htd.addFamily(new HColumnDescriptor(familyToAddName));
404     htd.removeFamily(familyToRemove.getBytes());
405     htd.setRegionReplication(3);
406 
407     // Start the Modify procedure && kill the executor
408     long procId = procExec.submitProcedure(
409       new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
410 
411     // Failing after MODIFY_TABLE_DELETE_FS_LAYOUT we should not trigger the rollback.
412     // NOTE: the 5 (number of MODIFY_TABLE_DELETE_FS_LAYOUT + 1 step) is hardcoded,
413     //       so you have to look at this test at least once when you add a new step.
414     int numberOfSteps = 5;
415     MasterProcedureTestingUtility.testRollbackAndDoubleExecutionAfterPONR(
416       procExec,
417       procId,
418       numberOfSteps,
419       ModifyTableState.values());
420 
421     // "cf2" should be added and "cf1" should be removed
422     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
423       tableName, regions, false, familyToAddName);
424   }
425 
426   private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
427     return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
428   }
429 }