001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.compactions;
019
020import static org.apache.hadoop.hbase.regionserver.StoreFileWriter.shouldEnableHistoricalCompactionFiles;
021
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.hbase.HConstants;
024import org.apache.hadoop.hbase.client.RegionInfo;
025import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
026import org.apache.hadoop.util.StringUtils;
027import org.apache.yetus.audience.InterfaceAudience;
028import org.slf4j.Logger;
029import org.slf4j.LoggerFactory;
030
031/**
032 * <p>
033 * Compaction configuration for a particular instance of HStore. Takes into account both global
034 * settings and ones set on the column family/store. Control knobs for default compaction algorithm:
035 * </p>
036 * <p>
037 * maxCompactSize - upper bound on file size to be included in minor compactions minCompactSize -
038 * lower bound below which compaction is selected without ratio test minFilesToCompact - lower bound
039 * on number of files in any minor compaction maxFilesToCompact - upper bound on number of files in
040 * any minor compaction compactionRatio - Ratio used for compaction minLocalityToForceCompact -
041 * Locality threshold for a store file to major compact (HBASE-11195)
042 * </p>
043 * Set parameter as "hbase.hstore.compaction.&lt;attribute&gt;"
044 */
045
046@InterfaceAudience.Private
047public class CompactionConfiguration {
048
049  private static final Logger LOG = LoggerFactory.getLogger(CompactionConfiguration.class);
050
051  public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio";
052  public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
053    "hbase.hstore.compaction.ratio.offpeak";
054  public static final String HBASE_HSTORE_COMPACTION_MIN_KEY_OLD =
055    "hbase.hstore.compactionThreshold";
056  public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min";
057  public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
058    "hbase.hstore.compaction.min.size";
059  public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
060  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
061    "hbase.hstore.compaction.max.size";
062  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY =
063    "hbase.hstore.compaction.max.size.offpeak";
064  public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour";
065  public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour";
066  public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
067    "hbase.hstore.min.locality.to.skip.major.compact";
068
069  public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT =
070    "hbase.hfile.compaction.discharger.thread.count";
071
072  /*
073   * The epoch time length for the windows we no longer compact
074   */
075  public static final String DATE_TIERED_MAX_AGE_MILLIS_KEY =
076    "hbase.hstore.compaction.date.tiered.max.storefile.age.millis";
077  public static final String DATE_TIERED_INCOMING_WINDOW_MIN_KEY =
078    "hbase.hstore.compaction.date.tiered.incoming.window.min";
079  public static final String COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY =
080    "hbase.hstore.compaction.date.tiered.window.policy.class";
081  public static final String DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY =
082    "hbase.hstore.compaction.date.tiered.single.output.for.minor.compaction";
083
084  private static final Class<
085    ? extends RatioBasedCompactionPolicy> DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS =
086      ExploringCompactionPolicy.class;
087
088  public static final String DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY =
089    "hbase.hstore.compaction.date.tiered.window.factory.class";
090
091  private static final Class<
092    ? extends CompactionWindowFactory> DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS =
093      ExponentialCompactionWindowFactory.class;
094
095  public static final String DATE_TIERED_STORAGE_POLICY_ENABLE_KEY =
096    "hbase.hstore.compaction.date.tiered.storage.policy.enable";
097  public static final String DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY =
098    "hbase.hstore.compaction.date.tiered.hot.window.age.millis";
099  public static final String DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY =
100    "hbase.hstore.compaction.date.tiered.hot.window.storage.policy";
101  public static final String DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY =
102    "hbase.hstore.compaction.date.tiered.warm.window.age.millis";
103  public static final String DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY =
104    "hbase.hstore.compaction.date.tiered.warm.window.storage.policy";
105  /** Windows older than warm age belong to COLD_WINDOW **/
106  public static final String DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY =
107    "hbase.hstore.compaction.date.tiered.cold.window.storage.policy";
108
109  Configuration conf;
110  StoreConfigInformation storeConfigInfo;
111
112  private final double offPeakCompactionRatio;
113  /** Since all these properties can change online, they are volatile **/
114  private final long maxCompactSize;
115  private final long offPeakMaxCompactSize;
116  private final long minCompactSize;
117  /** This one can be update **/
118  private int minFilesToCompact;
119  private final int maxFilesToCompact;
120  private final double compactionRatio;
121  private final long throttlePoint;
122  private final long majorCompactionPeriod;
123  private final float majorCompactionJitter;
124  private final float minLocalityToForceCompact;
125  private final long dateTieredMaxStoreFileAgeMillis;
126  private final int dateTieredIncomingWindowMin;
127  private final String compactionPolicyForDateTieredWindow;
128  private final boolean dateTieredSingleOutputForMinorCompaction;
129  private final String dateTieredCompactionWindowFactory;
130  private final boolean dateTieredStoragePolicyEnable;
131  private long hotWindowAgeMillis;
132  private long warmWindowAgeMillis;
133  private String hotWindowStoragePolicy;
134  private String warmWindowStoragePolicy;
135  private String coldWindowStoragePolicy;
136
137  CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) {
138    this.conf = conf;
139    this.storeConfigInfo = storeConfigInfo;
140
141    maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
142    offPeakMaxCompactSize =
143      conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, maxCompactSize);
144    minCompactSize =
145      conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, storeConfigInfo.getMemStoreFlushSize());
146    minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
147      conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY_OLD, 3)));
148    if (shouldEnableHistoricalCompactionFiles(conf)) {
149      // If historical file writing is enabled, we bump up the min value by one as DualFileWriter
150      // compacts files into two files, live and historical, instead of one. This also eliminates
151      // infinite re-compaction when the min value is set to 2
152      minFilesToCompact += 1;
153    }
154    maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
155    compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
156    offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
157
158    throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
159      2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize());
160    majorCompactionPeriod =
161      conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD);
162    majorCompactionJitter =
163      conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER, HConstants.DEFAULT_MAJOR_COMPACTION_JITTER);
164    minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f);
165
166    dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE);
167    dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6);
168    compactionPolicyForDateTieredWindow =
169      conf.get(COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY,
170        DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName());
171    dateTieredSingleOutputForMinorCompaction =
172      conf.getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true);
173    this.dateTieredCompactionWindowFactory =
174      conf.get(DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY,
175        DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName());
176    // for Heterogeneous Storage
177    dateTieredStoragePolicyEnable = conf.getBoolean(DATE_TIERED_STORAGE_POLICY_ENABLE_KEY, false);
178    hotWindowAgeMillis = conf.getLong(DATE_TIERED_HOT_WINDOW_AGE_MILLIS_KEY, 86400000L);
179    hotWindowStoragePolicy = conf.get(DATE_TIERED_HOT_WINDOW_STORAGE_POLICY_KEY, "ALL_SSD");
180    warmWindowAgeMillis = conf.getLong(DATE_TIERED_WARM_WINDOW_AGE_MILLIS_KEY, 604800000L);
181    warmWindowStoragePolicy = conf.get(DATE_TIERED_WARM_WINDOW_STORAGE_POLICY_KEY, "ONE_SSD");
182    coldWindowStoragePolicy = conf.get(DATE_TIERED_COLD_WINDOW_STORAGE_POLICY_KEY, "HOT");
183    LOG.info(toString());
184  }
185
186  @Override
187  public String toString() {
188    return String.format(
189      "size [minCompactSize:%s, maxCompactSize:%s, offPeakMaxCompactSize:%s);"
190        + " files [minFilesToCompact:%d, maxFilesToCompact:%d);"
191        + " ratio %f; off-peak ratio %f; throttle point %d;"
192        + " major period %d, major jitter %f, min locality to compact %f;"
193        + " tiered compaction: max_age %d, incoming window min %d,"
194        + " compaction policy for tiered window %s, single output for minor %b,"
195        + " compaction window factory %s," + " region %s columnFamilyName %s",
196      StringUtils.byteDesc(minCompactSize), StringUtils.byteDesc(maxCompactSize),
197      StringUtils.byteDesc(offPeakMaxCompactSize), minFilesToCompact, maxFilesToCompact,
198      compactionRatio, offPeakCompactionRatio, throttlePoint, majorCompactionPeriod,
199      majorCompactionJitter, minLocalityToForceCompact, dateTieredMaxStoreFileAgeMillis,
200      dateTieredIncomingWindowMin, compactionPolicyForDateTieredWindow,
201      dateTieredSingleOutputForMinorCompaction, dateTieredCompactionWindowFactory,
202      RegionInfo.prettyPrint(storeConfigInfo.getRegionInfo().getEncodedName()),
203      storeConfigInfo.getColumnFamilyName());
204  }
205
206  /** Returns lower bound below which compaction is selected without ratio test */
207  public long getMinCompactSize() {
208    return minCompactSize;
209  }
210
211  /** Returns upper bound on file size to be included in minor compactions */
212  public long getMaxCompactSize() {
213    return maxCompactSize;
214  }
215
216  /** Returns lower bound on number of files to be included in minor compactions */
217  public int getMinFilesToCompact() {
218    return minFilesToCompact;
219  }
220
221  /**
222   * Set lower bound on number of files to be included in minor compactions
223   * @param threshold value to set to
224   */
225  public void setMinFilesToCompact(int threshold) {
226    minFilesToCompact = threshold;
227  }
228
229  /** Returns upper bound on number of files to be included in minor compactions */
230  public int getMaxFilesToCompact() {
231    return maxFilesToCompact;
232  }
233
234  /** Returns Ratio used for compaction */
235  public double getCompactionRatio() {
236    return compactionRatio;
237  }
238
239  /** Returns Off peak Ratio used for compaction */
240  public double getCompactionRatioOffPeak() {
241    return offPeakCompactionRatio;
242  }
243
244  /** Returns ThrottlePoint used for classifying small and large compactions */
245  public long getThrottlePoint() {
246    return throttlePoint;
247  }
248
249  /**
250   * @return Major compaction period from compaction. Major compactions are selected periodically
251   *         according to this parameter plus jitter
252   */
253  public long getMajorCompactionPeriod() {
254    return majorCompactionPeriod;
255  }
256
257  /**
258   * @return Major the jitter fraction, the fraction within which the major compaction period is
259   *         randomly chosen from the majorCompactionPeriod in each store.
260   */
261  public float getMajorCompactionJitter() {
262    return majorCompactionJitter;
263  }
264
265  /**
266   * @return Block locality ratio, the ratio at which we will include old regions with a single
267   *         store file for major compaction. Used to improve block locality for regions that
268   *         haven't had writes in a while but are still being read.
269   */
270  public float getMinLocalityToForceCompact() {
271    return minLocalityToForceCompact;
272  }
273
274  public long getOffPeakMaxCompactSize() {
275    return offPeakMaxCompactSize;
276  }
277
278  public long getMaxCompactSize(boolean mayUseOffpeak) {
279    if (mayUseOffpeak) {
280      return getOffPeakMaxCompactSize();
281    } else {
282      return getMaxCompactSize();
283    }
284  }
285
286  public long getDateTieredMaxStoreFileAgeMillis() {
287    return dateTieredMaxStoreFileAgeMillis;
288  }
289
290  public int getDateTieredIncomingWindowMin() {
291    return dateTieredIncomingWindowMin;
292  }
293
294  public String getCompactionPolicyForDateTieredWindow() {
295    return compactionPolicyForDateTieredWindow;
296  }
297
298  public boolean useDateTieredSingleOutputForMinorCompaction() {
299    return dateTieredSingleOutputForMinorCompaction;
300  }
301
302  public String getDateTieredCompactionWindowFactory() {
303    return dateTieredCompactionWindowFactory;
304  }
305
306  public boolean isDateTieredStoragePolicyEnable() {
307    return dateTieredStoragePolicyEnable;
308  }
309
310  public long getHotWindowAgeMillis() {
311    return hotWindowAgeMillis;
312  }
313
314  public long getWarmWindowAgeMillis() {
315    return warmWindowAgeMillis;
316  }
317
318  public String getHotWindowStoragePolicy() {
319    return hotWindowStoragePolicy.trim().toUpperCase();
320  }
321
322  public String getWarmWindowStoragePolicy() {
323    return warmWindowStoragePolicy.trim().toUpperCase();
324  }
325
326  public String getColdWindowStoragePolicy() {
327    return coldWindowStoragePolicy.trim().toUpperCase();
328  }
329}