Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.zookeeper;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertTrue;
023
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Collections;
027import java.util.LinkedList;
028import java.util.List;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.Abortable;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseZKTestingUtil;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.apache.hadoop.hbase.testclassification.ZKTests;
035import org.apache.hadoop.hbase.util.Bytes;
036import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
037import org.apache.zookeeper.CreateMode;
038import org.apache.zookeeper.KeeperException;
039import org.apache.zookeeper.Op;
040import org.apache.zookeeper.ZooDefs.Ids;
041import org.junit.AfterClass;
042import org.junit.BeforeClass;
043import org.junit.ClassRule;
044import org.junit.Test;
045import org.junit.experimental.categories.Category;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049/**
050 * Test ZooKeeper multi-update functionality.
051 */
052@Category({ ZKTests.class, MediumTests.class })
053public class TestZKMulti {
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(TestZKMulti.class);
057
058  private static final Logger LOG = LoggerFactory.getLogger(TestZKMulti.class);
059  private final static HBaseZKTestingUtil TEST_UTIL = new HBaseZKTestingUtil();
060  private static ZKWatcher zkw = null;
061
062  private static class ZKMultiAbortable implements Abortable {
063    @Override
064    public void abort(String why, Throwable e) {
065      LOG.info(why, e);
066    }
067
068    @Override
069    public boolean isAborted() {
070      return false;
071    }
072  }
073
074  @BeforeClass
075  public static void setUpBeforeClass() throws Exception {
076    TEST_UTIL.startMiniZKCluster();
077    Configuration conf = TEST_UTIL.getConfiguration();
078    Abortable abortable = new ZKMultiAbortable();
079    zkw = new ZKWatcher(conf, "TestZKMulti", abortable, true);
080  }
081
082  @AfterClass
083  public static void tearDownAfterClass() throws Exception {
084    TEST_UTIL.shutdownMiniZKCluster();
085  }
086
087  @Test
088  public void testSimpleMulti() throws Exception {
089    // null multi
090    ZKUtil.multiOrSequential(zkw, null, false);
091
092    // empty multi
093    ZKUtil.multiOrSequential(zkw, new LinkedList<>(), false);
094
095    // single create
096    String path = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSimpleMulti");
097    LinkedList<ZKUtilOp> singleCreate = new LinkedList<>();
098    singleCreate.add(ZKUtilOp.createAndFailSilent(path, new byte[0]));
099    ZKUtil.multiOrSequential(zkw, singleCreate, false);
100    assertTrue(ZKUtil.checkExists(zkw, path) != -1);
101
102    // single setdata
103    LinkedList<ZKUtilOp> singleSetData = new LinkedList<>();
104    byte[] data = Bytes.toBytes("foobar");
105    singleSetData.add(ZKUtilOp.setData(path, data));
106    ZKUtil.multiOrSequential(zkw, singleSetData, false);
107    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path), data));
108
109    // single delete
110    LinkedList<ZKUtilOp> singleDelete = new LinkedList<>();
111    singleDelete.add(ZKUtilOp.deleteNodeFailSilent(path));
112    ZKUtil.multiOrSequential(zkw, singleDelete, false);
113    assertEquals(-1, ZKUtil.checkExists(zkw, path));
114  }
115
116  @Test
117  public void testComplexMulti() throws Exception {
118    String path1 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testComplexMulti1");
119    String path2 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testComplexMulti2");
120    String path3 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testComplexMulti3");
121    String path4 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testComplexMulti4");
122    String path5 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testComplexMulti5");
123    String path6 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testComplexMulti6");
124    // create 4 nodes that we'll setData on or delete later
125    LinkedList<ZKUtilOp> create4Nodes = new LinkedList<>();
126    create4Nodes.add(ZKUtilOp.createAndFailSilent(path1, Bytes.toBytes(path1)));
127    create4Nodes.add(ZKUtilOp.createAndFailSilent(path2, Bytes.toBytes(path2)));
128    create4Nodes.add(ZKUtilOp.createAndFailSilent(path3, Bytes.toBytes(path3)));
129    create4Nodes.add(ZKUtilOp.createAndFailSilent(path4, Bytes.toBytes(path4)));
130    ZKUtil.multiOrSequential(zkw, create4Nodes, false);
131    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), Bytes.toBytes(path1)));
132    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path2), Bytes.toBytes(path2)));
133    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path3), Bytes.toBytes(path3)));
134    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path4), Bytes.toBytes(path4)));
135
136    // do multiple of each operation (setData, delete, create)
137    LinkedList<ZKUtilOp> ops = new LinkedList<>();
138    // setData
139    ops.add(ZKUtilOp.setData(path1, Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1))));
140    ops.add(ZKUtilOp.setData(path2, Bytes.add(Bytes.toBytes(path2), Bytes.toBytes(path2))));
141    // delete
142    ops.add(ZKUtilOp.deleteNodeFailSilent(path3));
143    ops.add(ZKUtilOp.deleteNodeFailSilent(path4));
144    // create
145    ops.add(ZKUtilOp.createAndFailSilent(path5, Bytes.toBytes(path5)));
146    ops.add(ZKUtilOp.createAndFailSilent(path6, Bytes.toBytes(path6)));
147    ZKUtil.multiOrSequential(zkw, ops, false);
148    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1),
149      Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1))));
150    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path2),
151      Bytes.add(Bytes.toBytes(path2), Bytes.toBytes(path2))));
152    assertEquals(-1, ZKUtil.checkExists(zkw, path3));
153    assertEquals(-1, ZKUtil.checkExists(zkw, path4));
154    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path5), Bytes.toBytes(path5)));
155    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path6), Bytes.toBytes(path6)));
156  }
157
158  @Test
159  public void testSingleFailure() throws Exception {
160    // try to delete a node that doesn't exist
161    boolean caughtNoNode = false;
162    String path = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureZ");
163    LinkedList<ZKUtilOp> ops = new LinkedList<>();
164    ops.add(ZKUtilOp.deleteNodeFailSilent(path));
165    try {
166      ZKUtil.multiOrSequential(zkw, ops, false);
167    } catch (KeeperException.NoNodeException nne) {
168      caughtNoNode = true;
169    }
170    assertTrue(caughtNoNode);
171
172    // try to setData on a node that doesn't exist
173    caughtNoNode = false;
174    ops = new LinkedList<>();
175    ops.add(ZKUtilOp.setData(path, Bytes.toBytes(path)));
176    try {
177      ZKUtil.multiOrSequential(zkw, ops, false);
178    } catch (KeeperException.NoNodeException nne) {
179      caughtNoNode = true;
180    }
181    assertTrue(caughtNoNode);
182
183    // try to create on a node that already exists
184    boolean caughtNodeExists = false;
185    ops = new LinkedList<>();
186    ops.add(ZKUtilOp.createAndFailSilent(path, Bytes.toBytes(path)));
187    ZKUtil.multiOrSequential(zkw, ops, false);
188    try {
189      ZKUtil.multiOrSequential(zkw, ops, false);
190    } catch (KeeperException.NodeExistsException nee) {
191      caughtNodeExists = true;
192    }
193    assertTrue(caughtNodeExists);
194  }
195
196  @Test
197  public void testSingleFailureInMulti() throws Exception {
198    // try a multi where all but one operation succeeds
199    String pathA = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiA");
200    String pathB = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiB");
201    String pathC = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiC");
202    LinkedList<ZKUtilOp> ops = new LinkedList<>();
203    ops.add(ZKUtilOp.createAndFailSilent(pathA, Bytes.toBytes(pathA)));
204    ops.add(ZKUtilOp.createAndFailSilent(pathB, Bytes.toBytes(pathB)));
205    ops.add(ZKUtilOp.deleteNodeFailSilent(pathC));
206    boolean caughtNoNode = false;
207    try {
208      ZKUtil.multiOrSequential(zkw, ops, false);
209    } catch (KeeperException.NoNodeException nne) {
210      caughtNoNode = true;
211    }
212    assertTrue(caughtNoNode);
213    // assert that none of the operations succeeded
214    assertEquals(-1, ZKUtil.checkExists(zkw, pathA));
215    assertEquals(-1, ZKUtil.checkExists(zkw, pathB));
216    assertEquals(-1, ZKUtil.checkExists(zkw, pathC));
217  }
218
219  @Test
220  public void testMultiFailure() throws Exception {
221    String pathX = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testMultiFailureX");
222    String pathY = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testMultiFailureY");
223    String pathZ = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testMultiFailureZ");
224    // create X that we will use to fail create later
225    LinkedList<ZKUtilOp> ops = new LinkedList<>();
226    ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX)));
227    ZKUtil.multiOrSequential(zkw, ops, false);
228
229    // fail one of each create ,setData, delete
230    String pathV = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testMultiFailureV");
231    String pathW = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testMultiFailureW");
232    ops = new LinkedList<>();
233    ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX))); // fail -- already exists
234    ops.add(ZKUtilOp.setData(pathY, Bytes.toBytes(pathY))); // fail -- doesn't exist
235    ops.add(ZKUtilOp.deleteNodeFailSilent(pathZ)); // fail -- doesn't exist
236    ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathV))); // pass
237    ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathW))); // pass
238    boolean caughtNodeExists = false;
239    try {
240      ZKUtil.multiOrSequential(zkw, ops, false);
241    } catch (KeeperException.NodeExistsException nee) {
242      // check first operation that fails throws exception
243      caughtNodeExists = true;
244    }
245    assertTrue(caughtNodeExists);
246    // check that no modifications were made
247    assertNotEquals(-1, ZKUtil.checkExists(zkw, pathX));
248    assertEquals(-1, ZKUtil.checkExists(zkw, pathY));
249    assertEquals(-1, ZKUtil.checkExists(zkw, pathZ));
250    assertEquals(-1, ZKUtil.checkExists(zkw, pathW));
251    assertEquals(-1, ZKUtil.checkExists(zkw, pathV));
252
253    // test that with multiple failures, throws an exception corresponding to first failure in list
254    ops = new LinkedList<>();
255    ops.add(ZKUtilOp.setData(pathY, Bytes.toBytes(pathY))); // fail -- doesn't exist
256    ops.add(ZKUtilOp.createAndFailSilent(pathX, Bytes.toBytes(pathX))); // fail -- exists
257    boolean caughtNoNode = false;
258    try {
259      ZKUtil.multiOrSequential(zkw, ops, false);
260    } catch (KeeperException.NoNodeException nne) {
261      // check first operation that fails throws exception
262      caughtNoNode = true;
263    }
264    assertTrue(caughtNoNode);
265    // check that no modifications were made
266    assertNotEquals(-1, ZKUtil.checkExists(zkw, pathX));
267    assertEquals(-1, ZKUtil.checkExists(zkw, pathY));
268    assertEquals(-1, ZKUtil.checkExists(zkw, pathZ));
269    assertEquals(-1, ZKUtil.checkExists(zkw, pathW));
270    assertEquals(-1, ZKUtil.checkExists(zkw, pathV));
271  }
272
273  @Test
274  public void testRunSequentialOnMultiFailure() throws Exception {
275    String path1 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential1");
276    String path2 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential2");
277    String path3 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential3");
278    String path4 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential4");
279
280    // create some nodes that we will use later
281    LinkedList<ZKUtilOp> ops = new LinkedList<>();
282    ops.add(ZKUtilOp.createAndFailSilent(path1, Bytes.toBytes(path1)));
283    ops.add(ZKUtilOp.createAndFailSilent(path2, Bytes.toBytes(path2)));
284    ZKUtil.multiOrSequential(zkw, ops, false);
285
286    // test that, even with operations that fail, the ones that would pass will pass
287    // with runSequentialOnMultiFailure
288    ops = new LinkedList<>();
289    // pass
290    ops.add(ZKUtilOp.setData(path1, Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1))));
291    // pass
292    ops.add(ZKUtilOp.deleteNodeFailSilent(path2));
293    // fail -- node doesn't exist
294    ops.add(ZKUtilOp.deleteNodeFailSilent(path3));
295    // pass
296    ops.add(
297      ZKUtilOp.createAndFailSilent(path4, Bytes.add(Bytes.toBytes(path4), Bytes.toBytes(path4))));
298    ZKUtil.multiOrSequential(zkw, ops, true);
299    assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1),
300      Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1))));
301    assertEquals(-1, ZKUtil.checkExists(zkw, path2));
302    assertEquals(-1, ZKUtil.checkExists(zkw, path3));
303    assertNotEquals(-1, ZKUtil.checkExists(zkw, path4));
304  }
305
306  /**
307   * Verifies that for the given root node, it should delete all the child nodes recursively using
308   * multi-update api.
309   */
310  @Test
311  public void testdeleteChildrenRecursivelyMulti() throws Exception {
312    String parentZNode = "/testRootMulti";
313    createZNodeTree(parentZNode);
314
315    ZKUtil.deleteChildrenRecursivelyMultiOrSequential(zkw, true, parentZNode);
316
317    assertTrue("Wrongly deleted parent znode!", ZKUtil.checkExists(zkw, parentZNode) > -1);
318    List<String> children = zkw.getRecoverableZooKeeper().getChildren(parentZNode, false);
319    assertEquals("Failed to delete child znodes!", 0, children.size());
320  }
321
322  /**
323   * Verifies that for the given root node, it should delete all the nodes recursively using
324   * multi-update api.
325   */
326  @Test
327  public void testDeleteNodeRecursivelyMulti() throws Exception {
328    String parentZNode = "/testdeleteNodeRecursivelyMulti";
329    createZNodeTree(parentZNode);
330
331    ZKUtil.deleteNodeRecursively(zkw, parentZNode);
332    assertEquals("Parent znode should be deleted.", -1, ZKUtil.checkExists(zkw, parentZNode));
333  }
334
335  @Test
336  public void testDeleteNodeRecursivelyMultiOrSequential() throws Exception {
337    String parentZNode1 = "/testdeleteNode1";
338    String parentZNode2 = "/testdeleteNode2";
339    String parentZNode3 = "/testdeleteNode3";
340    createZNodeTree(parentZNode1);
341    createZNodeTree(parentZNode2);
342    createZNodeTree(parentZNode3);
343
344    ZKUtil.deleteNodeRecursivelyMultiOrSequential(zkw, false, parentZNode1, parentZNode2,
345      parentZNode3);
346    assertEquals("Parent znode 1 should be deleted.", -1, ZKUtil.checkExists(zkw, parentZNode1));
347    assertEquals("Parent znode 2 should be deleted.", -1, ZKUtil.checkExists(zkw, parentZNode2));
348    assertEquals("Parent znode 3 should be deleted.", -1, ZKUtil.checkExists(zkw, parentZNode3));
349  }
350
351  @Test
352  public void testDeleteChildrenRecursivelyMultiOrSequential() throws Exception {
353    String parentZNode1 = "/testdeleteChildren1";
354    String parentZNode2 = "/testdeleteChildren2";
355    String parentZNode3 = "/testdeleteChildren3";
356    createZNodeTree(parentZNode1);
357    createZNodeTree(parentZNode2);
358    createZNodeTree(parentZNode3);
359
360    ZKUtil.deleteChildrenRecursivelyMultiOrSequential(zkw, true, parentZNode1, parentZNode2,
361      parentZNode3);
362
363    assertTrue("Wrongly deleted parent znode 1!", ZKUtil.checkExists(zkw, parentZNode1) > -1);
364    List<String> children = zkw.getRecoverableZooKeeper().getChildren(parentZNode1, false);
365    assertEquals("Failed to delete child znodes of parent znode 1!", 0, children.size());
366
367    assertTrue("Wrongly deleted parent znode 2!", ZKUtil.checkExists(zkw, parentZNode2) > -1);
368    children = zkw.getRecoverableZooKeeper().getChildren(parentZNode2, false);
369    assertEquals("Failed to delete child znodes of parent znode 1!", 0, children.size());
370
371    assertTrue("Wrongly deleted parent znode 3!", ZKUtil.checkExists(zkw, parentZNode3) > -1);
372    children = zkw.getRecoverableZooKeeper().getChildren(parentZNode3, false);
373    assertEquals("Failed to delete child znodes of parent znode 1!", 0, children.size());
374  }
375
376  @Test
377  public void testBatchedDeletesOfWideZNodes() throws Exception {
378    // Batch every 50bytes
379    final int batchSize = 50;
380    Configuration localConf = new Configuration(TEST_UTIL.getConfiguration());
381    localConf.setInt("zookeeper.multi.max.size", batchSize);
382    try (ZKWatcher customZkw =
383      new ZKWatcher(localConf, "TestZKMulti_Custom", new ZKMultiAbortable(), true)) {
384
385      // With a parent znode like this, we'll get batches of 2-3 elements
386      final String parent1 = "/batchedDeletes1";
387      final String parent2 = "/batchedDeletes2";
388      final byte[] EMPTY_BYTES = new byte[0];
389
390      // Write one node
391      List<Op> ops = new ArrayList<>();
392      ops.add(Op.create(parent1, EMPTY_BYTES, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
393      for (int i = 0; i < batchSize * 2; i++) {
394        ops.add(
395          Op.create(parent1 + "/" + i, EMPTY_BYTES, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
396      }
397      customZkw.getRecoverableZooKeeper().multi(ops);
398
399      // Write into a second node
400      ops.clear();
401      ops.add(Op.create(parent2, EMPTY_BYTES, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
402      for (int i = 0; i < batchSize * 4; i++) {
403        ops.add(
404          Op.create(parent2 + "/" + i, EMPTY_BYTES, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
405      }
406      customZkw.getRecoverableZooKeeper().multi(ops);
407
408      // These should return successfully
409      ZKUtil.deleteChildrenRecursively(customZkw, parent1);
410      ZKUtil.deleteChildrenRecursively(customZkw, parent2);
411    }
412  }
413
414  @Test
415  public void testListPartitioning() {
416    // 10 Bytes
417    ZKUtilOp tenByteOp = ZKUtilOp.deleteNodeFailSilent("/123456789");
418
419    // Simple, single element case
420    assertEquals(Collections.singletonList(Collections.singletonList(tenByteOp)),
421      ZKUtil.partitionOps(Collections.singletonList(tenByteOp), 15));
422
423    // Simple case where we exceed the limit, but must make the list
424    assertEquals(Collections.singletonList(Collections.singletonList(tenByteOp)),
425      ZKUtil.partitionOps(Collections.singletonList(tenByteOp), 5));
426
427    // Each gets its own bucket
428    assertEquals(
429      Arrays.asList(Collections.singletonList(tenByteOp), Collections.singletonList(tenByteOp),
430        Collections.singletonList(tenByteOp)),
431      ZKUtil.partitionOps(Arrays.asList(tenByteOp, tenByteOp, tenByteOp), 15));
432
433    // Test internal boundary
434    assertEquals(
435      Arrays.asList(Arrays.asList(tenByteOp, tenByteOp), Collections.singletonList(tenByteOp)),
436      ZKUtil.partitionOps(Arrays.asList(tenByteOp, tenByteOp, tenByteOp), 20));
437
438    // Plenty of space for one partition
439    assertEquals(Collections.singletonList(Arrays.asList(tenByteOp, tenByteOp, tenByteOp)),
440      ZKUtil.partitionOps(Arrays.asList(tenByteOp, tenByteOp, tenByteOp), 50));
441  }
442
443  private void createZNodeTree(String rootZNode) throws KeeperException, InterruptedException {
444    List<Op> opList = new ArrayList<>();
445    opList.add(Op.create(rootZNode, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
446    int level = 0;
447    String parentZNode = rootZNode;
448    while (level < 10) {
449      // define parent node
450      parentZNode = parentZNode + "/" + level;
451      opList.add(Op.create(parentZNode, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT));
452      int elements = 0;
453      // add elements to the parent node
454      while (elements < level) {
455        opList.add(Op.create(parentZNode + "/" + elements, new byte[0], Ids.OPEN_ACL_UNSAFE,
456          CreateMode.PERSISTENT));
457        elements++;
458      }
459      level++;
460    }
461    zkw.getRecoverableZooKeeper().multi(opList);
462  }
463}