001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.replication;
019
020import java.io.IOException;
021import org.apache.hadoop.hbase.HBaseClassTestRule;
022import org.apache.hadoop.hbase.HBaseTestingUtility;
023import org.apache.hadoop.hbase.ProcedureTestUtil;
024import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
025import org.apache.hadoop.hbase.procedure2.Procedure;
026import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
027import org.apache.hadoop.hbase.replication.ReplicationException;
028import org.apache.hadoop.hbase.testclassification.LargeTests;
029import org.apache.hadoop.hbase.testclassification.MasterTests;
030import org.junit.AfterClass;
031import org.junit.BeforeClass;
032import org.junit.ClassRule;
033import org.junit.Test;
034import org.junit.experimental.categories.Category;
035
036import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationState;
037
038@Category({ MasterTests.class, LargeTests.class })
039public class TestModifyPeerProcedureRetryBackoff {
040
041  @ClassRule
042  public static final HBaseClassTestRule CLASS_RULE =
043    HBaseClassTestRule.forClass(TestModifyPeerProcedureRetryBackoff.class);
044
045  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
046
047  private static boolean FAIL = true;
048
049  public static class TestModifyPeerProcedure extends ModifyPeerProcedure {
050
051    public TestModifyPeerProcedure() {
052    }
053
054    public TestModifyPeerProcedure(String peerId) {
055      super(peerId);
056    }
057
058    @Override
059    public PeerOperationType getPeerOperationType() {
060      return PeerOperationType.ADD;
061    }
062
063    private void tryFail() throws ReplicationException {
064      synchronized (TestModifyPeerProcedureRetryBackoff.class) {
065        if (FAIL) {
066          throw new ReplicationException("Inject error");
067        }
068        FAIL = true;
069      }
070    }
071
072    @Override
073    protected <T extends Procedure<MasterProcedureEnv>> void
074      addChildProcedure(@SuppressWarnings("unchecked") T... subProcedure) {
075      // Make it a no-op
076    }
077
078    @Override
079    protected PeerModificationState nextStateAfterRefresh() {
080      return PeerModificationState.SERIAL_PEER_REOPEN_REGIONS;
081    }
082
083    @Override
084    protected boolean enablePeerBeforeFinish() {
085      return true;
086    }
087
088    @Override
089    protected void updateLastPushedSequenceIdForSerialPeer(MasterProcedureEnv env)
090      throws IOException, ReplicationException {
091      tryFail();
092    }
093
094    @Override
095    protected void reopenRegions(MasterProcedureEnv env) throws IOException {
096      try {
097        tryFail();
098      } catch (ReplicationException e) {
099        throw new IOException(e);
100      }
101    }
102
103    @Override
104    protected void enablePeer(MasterProcedureEnv env) throws ReplicationException {
105      tryFail();
106    }
107
108    @Override
109    protected void prePeerModification(MasterProcedureEnv env)
110      throws IOException, ReplicationException {
111      tryFail();
112    }
113
114    @Override
115    protected void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException {
116      tryFail();
117    }
118
119    @Override
120    protected void postPeerModification(MasterProcedureEnv env)
121      throws IOException, ReplicationException {
122      tryFail();
123    }
124  }
125
126  @BeforeClass
127  public static void setUp() throws Exception {
128    UTIL.startMiniCluster(1);
129  }
130
131  @AfterClass
132  public static void tearDown() throws Exception {
133    UTIL.shutdownMiniCluster();
134  }
135
136  private void assertBackoffIncrease() throws IOException, InterruptedException {
137    ProcedureTestUtil.waitUntilProcedureWaitingTimeout(UTIL, TestModifyPeerProcedure.class, 30000);
138    ProcedureTestUtil.waitUntilProcedureTimeoutIncrease(UTIL, TestModifyPeerProcedure.class, 2);
139    synchronized (TestModifyPeerProcedureRetryBackoff.class) {
140      FAIL = false;
141    }
142    UTIL.waitFor(30000, () -> FAIL);
143  }
144
145  @Test
146  public void test() throws IOException, InterruptedException {
147    ProcedureExecutor<MasterProcedureEnv> procExec =
148      UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
149    long procId = procExec.submitProcedure(new TestModifyPeerProcedure("1"));
150    // PRE_PEER_MODIFICATION
151    assertBackoffIncrease();
152    // UPDATE_PEER_STORAGE
153    assertBackoffIncrease();
154    // No retry for REFRESH_PEER_ON_RS
155    // SERIAL_PEER_REOPEN_REGIONS
156    assertBackoffIncrease();
157    // SERIAL_PEER_UPDATE_LAST_PUSHED_SEQ_ID
158    assertBackoffIncrease();
159    // SERIAL_PEER_SET_PEER_ENABLED
160    assertBackoffIncrease();
161    // No retry for SERIAL_PEER_ENABLE_PEER_REFRESH_PEER_ON_RS
162    // POST_PEER_MODIFICATION
163    assertBackoffIncrease();
164    UTIL.waitFor(30000, () -> procExec.isFinished(procId));
165  }
166}