001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import java.io.IOException;
021import java.io.InputStream;
022import java.util.ArrayList;
023import java.util.Calendar;
024import java.util.Date;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Map;
028import java.util.Map.Entry;
029import java.util.Set;
030import org.apache.commons.lang3.StringUtils;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.backup.util.BackupUtils;
033import org.apache.hadoop.hbase.util.Bytes;
034import org.apache.yetus.audience.InterfaceAudience;
035import org.slf4j.Logger;
036import org.slf4j.LoggerFactory;
037
038import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
039import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
040
041/**
042 * An object to encapsulate the information for each backup session
043 */
044@InterfaceAudience.Private
045public class BackupInfo implements Comparable<BackupInfo> {
046  private static final Logger LOG = LoggerFactory.getLogger(BackupInfo.class);
047
048  public interface Filter {
049    /**
050     * Filter interface
051     * @param info backup info
052     * @return true if info passes filter, false otherwise
053     */
054    boolean apply(BackupInfo info);
055  }
056
057  /**
058   * Backup session states
059   */
060  public enum BackupState {
061    RUNNING,
062    COMPLETE,
063    FAILED,
064    ANY
065  }
066
067  /**
068   * BackupPhase - phases of an ACTIVE backup session (running), when state of a backup session is
069   * BackupState.RUNNING
070   */
071  public enum BackupPhase {
072    REQUEST,
073    SNAPSHOT,
074    PREPARE_INCREMENTAL,
075    SNAPSHOTCOPY,
076    INCREMENTAL_COPY,
077    STORE_MANIFEST
078  }
079
080  /**
081   * Backup id
082   */
083  private String backupId;
084
085  /**
086   * Backup type, full or incremental
087   */
088  private BackupType type;
089
090  /**
091   * Target root directory for storing the backup files
092   */
093  private String backupRootDir;
094
095  /**
096   * Backup state
097   */
098  private BackupState state;
099
100  /**
101   * Backup phase
102   */
103  private BackupPhase phase = BackupPhase.REQUEST;
104
105  /**
106   * Backup failure message
107   */
108  private String failedMsg;
109
110  /**
111   * Backup status map for all tables
112   */
113  private Map<TableName, BackupTableInfo> backupTableInfoMap;
114
115  /**
116   * Actual start timestamp of a backup process
117   */
118  private long startTs;
119
120  /**
121   * Actual end timestamp of the backup process
122   */
123  private long completeTs;
124
125  /**
126   * Total bytes of incremental logs copied
127   */
128  private long totalBytesCopied;
129
130  /**
131   * For incremental backup, a location of a backed-up hlogs
132   */
133  private String hlogTargetDir = null;
134
135  /**
136   * Incremental backup file list
137   */
138  private List<String> incrBackupFileList;
139
140  /**
141   * New region server log timestamps for table set after distributed log roll key - table name,
142   * value - map of RegionServer hostname -> last log rolled timestamp
143   */
144  private Map<TableName, Map<String, Long>> tableSetTimestampMap;
145
146  /**
147   * Previous Region server log timestamps for table set after distributed log roll key - table
148   * name, value - map of RegionServer hostname -> last log rolled timestamp
149   */
150  private Map<TableName, Map<String, Long>> incrTimestampMap;
151
152  /**
153   * Backup progress in %% (0-100)
154   */
155  private int progress;
156
157  /**
158   * Number of parallel workers. -1 - system defined
159   */
160  private int workers = -1;
161
162  /**
163   * Bandwidth per worker in MB per sec. -1 - unlimited
164   */
165  private long bandwidth = -1;
166
167  public BackupInfo() {
168    backupTableInfoMap = new HashMap<>();
169  }
170
171  public BackupInfo(String backupId, BackupType type, TableName[] tables, String targetRootDir) {
172    this();
173    this.backupId = backupId;
174    this.type = type;
175    this.backupRootDir = targetRootDir;
176    this.addTables(tables);
177    if (type == BackupType.INCREMENTAL) {
178      setHLogTargetDir(BackupUtils.getLogBackupDir(targetRootDir, backupId));
179    }
180    this.startTs = 0;
181    this.completeTs = 0;
182  }
183
184  public int getWorkers() {
185    return workers;
186  }
187
188  public void setWorkers(int workers) {
189    this.workers = workers;
190  }
191
192  public long getBandwidth() {
193    return bandwidth;
194  }
195
196  public void setBandwidth(long bandwidth) {
197    this.bandwidth = bandwidth;
198  }
199
200  public void setBackupTableInfoMap(Map<TableName, BackupTableInfo> backupTableInfoMap) {
201    this.backupTableInfoMap = backupTableInfoMap;
202  }
203
204  public Map<TableName, Map<String, Long>> getTableSetTimestampMap() {
205    return tableSetTimestampMap;
206  }
207
208  public void setTableSetTimestampMap(Map<TableName, Map<String, Long>> tableSetTimestampMap) {
209    this.tableSetTimestampMap = tableSetTimestampMap;
210  }
211
212  public void setType(BackupType type) {
213    this.type = type;
214  }
215
216  public void setBackupRootDir(String targetRootDir) {
217    this.backupRootDir = targetRootDir;
218  }
219
220  public void setTotalBytesCopied(long totalBytesCopied) {
221    this.totalBytesCopied = totalBytesCopied;
222  }
223
224  /**
225   * Set progress (0-100%)
226   * @param p progress value
227   */
228  public void setProgress(int p) {
229    this.progress = p;
230  }
231
232  /**
233   * Get current progress
234   */
235  public int getProgress() {
236    return progress;
237  }
238
239  public String getBackupId() {
240    return backupId;
241  }
242
243  public void setBackupId(String backupId) {
244    this.backupId = backupId;
245  }
246
247  public BackupTableInfo getBackupTableInfo(TableName table) {
248    return this.backupTableInfoMap.get(table);
249  }
250
251  public String getFailedMsg() {
252    return failedMsg;
253  }
254
255  public void setFailedMsg(String failedMsg) {
256    this.failedMsg = failedMsg;
257  }
258
259  public long getStartTs() {
260    return startTs;
261  }
262
263  public void setStartTs(long startTs) {
264    this.startTs = startTs;
265  }
266
267  public long getCompleteTs() {
268    return completeTs;
269  }
270
271  public void setCompleteTs(long endTs) {
272    this.completeTs = endTs;
273  }
274
275  public long getTotalBytesCopied() {
276    return totalBytesCopied;
277  }
278
279  public BackupState getState() {
280    return state;
281  }
282
283  public void setState(BackupState flag) {
284    this.state = flag;
285  }
286
287  public BackupPhase getPhase() {
288    return phase;
289  }
290
291  public void setPhase(BackupPhase phase) {
292    this.phase = phase;
293  }
294
295  public BackupType getType() {
296    return type;
297  }
298
299  public void setSnapshotName(TableName table, String snapshotName) {
300    this.backupTableInfoMap.get(table).setSnapshotName(snapshotName);
301  }
302
303  public String getSnapshotName(TableName table) {
304    return this.backupTableInfoMap.get(table).getSnapshotName();
305  }
306
307  public List<String> getSnapshotNames() {
308    List<String> snapshotNames = new ArrayList<>();
309    for (BackupTableInfo backupStatus : this.backupTableInfoMap.values()) {
310      snapshotNames.add(backupStatus.getSnapshotName());
311    }
312    return snapshotNames;
313  }
314
315  public Set<TableName> getTables() {
316    return this.backupTableInfoMap.keySet();
317  }
318
319  public List<TableName> getTableNames() {
320    return new ArrayList<>(backupTableInfoMap.keySet());
321  }
322
323  public void addTables(TableName[] tables) {
324    for (TableName table : tables) {
325      BackupTableInfo backupStatus = new BackupTableInfo(table, this.backupRootDir, this.backupId);
326      this.backupTableInfoMap.put(table, backupStatus);
327    }
328  }
329
330  public void setTables(List<TableName> tables) {
331    this.backupTableInfoMap.clear();
332    for (TableName table : tables) {
333      BackupTableInfo backupStatus = new BackupTableInfo(table, this.backupRootDir, this.backupId);
334      this.backupTableInfoMap.put(table, backupStatus);
335    }
336  }
337
338  public String getBackupRootDir() {
339    return backupRootDir;
340  }
341
342  public String getTableBackupDir(TableName tableName) {
343    return BackupUtils.getTableBackupDir(backupRootDir, backupId, tableName);
344  }
345
346  public void setHLogTargetDir(String hlogTagetDir) {
347    this.hlogTargetDir = hlogTagetDir;
348  }
349
350  public String getHLogTargetDir() {
351    return hlogTargetDir;
352  }
353
354  public List<String> getIncrBackupFileList() {
355    return incrBackupFileList;
356  }
357
358  public void setIncrBackupFileList(List<String> incrBackupFileList) {
359    this.incrBackupFileList = incrBackupFileList;
360  }
361
362  /**
363   * Set the new region server log timestamps after distributed log roll
364   * @param prevTableSetTimestampMap table timestamp map
365   */
366  public void setIncrTimestampMap(Map<TableName, Map<String, Long>> prevTableSetTimestampMap) {
367    this.incrTimestampMap = prevTableSetTimestampMap;
368  }
369
370  /**
371   * Get new region server log timestamps after distributed log roll
372   * @return new region server log timestamps
373   */
374  public Map<TableName, Map<String, Long>> getIncrTimestampMap() {
375    return this.incrTimestampMap;
376  }
377
378  public TableName getTableBySnapshot(String snapshotName) {
379    for (Entry<TableName, BackupTableInfo> entry : this.backupTableInfoMap.entrySet()) {
380      if (snapshotName.equals(entry.getValue().getSnapshotName())) {
381        return entry.getKey();
382      }
383    }
384    return null;
385  }
386
387  public BackupProtos.BackupInfo toProtosBackupInfo() {
388    BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder();
389    builder.setBackupId(getBackupId());
390    setBackupTableInfoMap(builder);
391    setTableSetTimestampMap(builder);
392    builder.setCompleteTs(getCompleteTs());
393    if (getFailedMsg() != null) {
394      builder.setFailedMessage(getFailedMsg());
395    }
396    if (getState() != null) {
397      builder.setBackupState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name()));
398    }
399    if (getPhase() != null) {
400      builder.setBackupPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name()));
401    }
402
403    builder.setProgress(getProgress());
404    builder.setStartTs(getStartTs());
405    builder.setBackupRootDir(getBackupRootDir());
406    builder.setBackupType(BackupProtos.BackupType.valueOf(getType().name()));
407    builder.setWorkersNumber(workers);
408    builder.setBandwidth(bandwidth);
409    return builder.build();
410  }
411
412  @Override
413  public int hashCode() {
414    int hash = 33 * type.hashCode() + backupId != null ? backupId.hashCode() : 0;
415    if (backupRootDir != null) {
416      hash = 33 * hash + backupRootDir.hashCode();
417    }
418    hash = 33 * hash + state.hashCode();
419    hash = 33 * hash + phase.hashCode();
420    hash = 33 * hash + (int) (startTs ^ (startTs >>> 32));
421    hash = 33 * hash + (int) (completeTs ^ (completeTs >>> 32));
422    hash = 33 * hash + (int) (totalBytesCopied ^ (totalBytesCopied >>> 32));
423    if (hlogTargetDir != null) {
424      hash = 33 * hash + hlogTargetDir.hashCode();
425    }
426    return hash;
427  }
428
429  @Override
430  public boolean equals(Object obj) {
431    if (obj instanceof BackupInfo) {
432      BackupInfo other = (BackupInfo) obj;
433      try {
434        return Bytes.equals(toByteArray(), other.toByteArray());
435      } catch (IOException e) {
436        LOG.error(e.toString(), e);
437        return false;
438      }
439    } else {
440      return false;
441    }
442  }
443
444  @Override
445  public String toString() {
446    return backupId;
447  }
448
449  public byte[] toByteArray() throws IOException {
450    return toProtosBackupInfo().toByteArray();
451  }
452
453  private void setBackupTableInfoMap(BackupProtos.BackupInfo.Builder builder) {
454    for (Entry<TableName, BackupTableInfo> entry : backupTableInfoMap.entrySet()) {
455      builder.addBackupTableInfo(entry.getValue().toProto());
456    }
457  }
458
459  private void setTableSetTimestampMap(BackupProtos.BackupInfo.Builder builder) {
460    if (this.getTableSetTimestampMap() != null) {
461      for (Entry<TableName, Map<String, Long>> entry : this.getTableSetTimestampMap().entrySet()) {
462        builder.putTableSetTimestamp(entry.getKey().getNameAsString(),
463          BackupProtos.BackupInfo.RSTimestampMap.newBuilder().putAllRsTimestamp(entry.getValue())
464            .build());
465      }
466    }
467  }
468
469  public static BackupInfo fromByteArray(byte[] data) throws IOException {
470    return fromProto(BackupProtos.BackupInfo.parseFrom(data));
471  }
472
473  public static BackupInfo fromStream(final InputStream stream) throws IOException {
474    return fromProto(BackupProtos.BackupInfo.parseDelimitedFrom(stream));
475  }
476
477  public static BackupInfo fromProto(BackupProtos.BackupInfo proto) {
478    BackupInfo context = new BackupInfo();
479    context.setBackupId(proto.getBackupId());
480    context.setBackupTableInfoMap(toMap(proto.getBackupTableInfoList()));
481    context.setTableSetTimestampMap(getTableSetTimestampMap(proto.getTableSetTimestampMap()));
482    context.setCompleteTs(proto.getCompleteTs());
483    if (proto.hasFailedMessage()) {
484      context.setFailedMsg(proto.getFailedMessage());
485    }
486    if (proto.hasBackupState()) {
487      context.setState(BackupInfo.BackupState.valueOf(proto.getBackupState().name()));
488    }
489
490    context
491      .setHLogTargetDir(BackupUtils.getLogBackupDir(proto.getBackupRootDir(), proto.getBackupId()));
492
493    if (proto.hasBackupPhase()) {
494      context.setPhase(BackupPhase.valueOf(proto.getBackupPhase().name()));
495    }
496    if (proto.hasProgress()) {
497      context.setProgress(proto.getProgress());
498    }
499    context.setStartTs(proto.getStartTs());
500    context.setBackupRootDir(proto.getBackupRootDir());
501    context.setType(BackupType.valueOf(proto.getBackupType().name()));
502    context.setWorkers(proto.getWorkersNumber());
503    context.setBandwidth(proto.getBandwidth());
504    return context;
505  }
506
507  private static Map<TableName, BackupTableInfo> toMap(List<BackupProtos.BackupTableInfo> list) {
508    HashMap<TableName, BackupTableInfo> map = new HashMap<>();
509    for (BackupProtos.BackupTableInfo tbs : list) {
510      map.put(ProtobufUtil.toTableName(tbs.getTableName()), BackupTableInfo.convert(tbs));
511    }
512    return map;
513  }
514
515  private static Map<TableName, Map<String, Long>>
516    getTableSetTimestampMap(Map<String, BackupProtos.BackupInfo.RSTimestampMap> map) {
517    Map<TableName, Map<String, Long>> tableSetTimestampMap = new HashMap<>();
518    for (Entry<String, BackupProtos.BackupInfo.RSTimestampMap> entry : map.entrySet()) {
519      tableSetTimestampMap.put(TableName.valueOf(entry.getKey()),
520        entry.getValue().getRsTimestampMap());
521    }
522
523    return tableSetTimestampMap;
524  }
525
526  public String getShortDescription() {
527    StringBuilder sb = new StringBuilder();
528    sb.append("{");
529    sb.append("ID=" + backupId).append(",");
530    sb.append("Type=" + getType()).append(",");
531    sb.append("Tables=" + getTableListAsString()).append(",");
532    sb.append("State=" + getState()).append(",");
533    Calendar cal = Calendar.getInstance();
534    cal.setTimeInMillis(getStartTs());
535    Date date = cal.getTime();
536    sb.append("Start time=" + date).append(",");
537    if (state == BackupState.FAILED) {
538      sb.append("Failed message=" + getFailedMsg()).append(",");
539    } else if (state == BackupState.RUNNING) {
540      sb.append("Phase=" + getPhase()).append(",");
541    } else if (state == BackupState.COMPLETE) {
542      cal = Calendar.getInstance();
543      cal.setTimeInMillis(getCompleteTs());
544      date = cal.getTime();
545      sb.append("End time=" + date).append(",");
546    }
547    sb.append("Progress=" + getProgress() + "%");
548    sb.append("}");
549
550    return sb.toString();
551  }
552
553  public String getStatusAndProgressAsString() {
554    StringBuilder sb = new StringBuilder();
555    sb.append("id: ").append(getBackupId()).append(" state: ").append(getState())
556      .append(" progress: ").append(getProgress());
557    return sb.toString();
558  }
559
560  public String getTableListAsString() {
561    StringBuilder sb = new StringBuilder();
562    sb.append("{");
563    sb.append(StringUtils.join(backupTableInfoMap.keySet(), ","));
564    sb.append("}");
565    return sb.toString();
566  }
567
568  /**
569   * We use only time stamps to compare objects during sort operation
570   */
571  @Override
572  public int compareTo(BackupInfo o) {
573    Long thisTS =
574      Long.valueOf(this.getBackupId().substring(this.getBackupId().lastIndexOf("_") + 1));
575    Long otherTS = Long.valueOf(o.getBackupId().substring(o.getBackupId().lastIndexOf("_") + 1));
576    return thisTS.compareTo(otherTS);
577  }
578}