001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.http;
019
020import java.io.File;
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.List;
024import java.util.concurrent.TimeUnit;
025import java.util.concurrent.atomic.AtomicInteger;
026import java.util.concurrent.locks.Lock;
027import java.util.concurrent.locks.ReentrantLock;
028import javax.servlet.http.HttpServlet;
029import javax.servlet.http.HttpServletRequest;
030import javax.servlet.http.HttpServletResponse;
031import org.apache.hadoop.hbase.util.ProcessUtils;
032import org.apache.yetus.audience.InterfaceAudience;
033import org.slf4j.Logger;
034import org.slf4j.LoggerFactory;
035
036import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
037
038/**
039 * Servlet that runs async-profiler as web-endpoint. Following options from async-profiler can be
040 * specified as query paramater. // -e event profiling event: cpu|alloc|lock|cache-misses etc. // -d
041 * duration run profiling for 'duration' seconds (integer) // -i interval sampling interval in
042 * nanoseconds (long) // -j jstackdepth maximum Java stack depth (integer) // -b bufsize frame
043 * buffer size (long) // -t profile different threads separately // -s simple class names instead of
044 * FQN // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr|html // --width
045 * px SVG width pixels (integer) // --height px SVG frame height pixels (integer) // --minwidth px
046 * skip frames smaller than px (double) // --reverse generate stack-reversed FlameGraph / Call tree
047 * Example: - To collect 30 second CPU profile of current process (returns FlameGraph svg) curl
048 * "http://localhost:10002/prof" - To collect 1 minute CPU profile of current process and output in
049 * tree format (html) curl "http://localhost:10002/prof?output=tree&duration=60" - To collect 30
050 * second heap allocation profile of current process (returns FlameGraph svg) curl
051 * "http://localhost:10002/prof?event=alloc" - To collect lock contention profile of current process
052 * (returns FlameGraph svg) curl "http://localhost:10002/prof?event=lock" Following event types are
053 * supported (default is 'cpu') (NOTE: not all OS'es support all events) // Perf events: // cpu //
054 * page-faults // context-switches // cycles // instructions // cache-references // cache-misses //
055 * branches // branch-misses // bus-cycles // L1-dcache-load-misses // LLC-load-misses //
056 * dTLB-load-misses // mem:breakpoint // trace:tracepoint // Java events: // alloc // lock
057 */
058@InterfaceAudience.Private
059public class ProfileServlet extends HttpServlet {
060
061  private static final long serialVersionUID = 1L;
062  private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class);
063
064  private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
065  private static final String ALLOWED_METHODS = "GET";
066  private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
067  private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8";
068  private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME";
069  private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home";
070  private static final String PROFILER_SCRIPT = "/profiler.sh";
071  private static final int DEFAULT_DURATION_SECONDS = 10;
072  private static final AtomicInteger ID_GEN = new AtomicInteger(0);
073  static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output-hbase";
074
075  enum Event {
076    CPU("cpu"),
077    WALL("wall"),
078    ALLOC("alloc"),
079    LOCK("lock"),
080    PAGE_FAULTS("page-faults"),
081    CONTEXT_SWITCHES("context-switches"),
082    CYCLES("cycles"),
083    INSTRUCTIONS("instructions"),
084    CACHE_REFERENCES("cache-references"),
085    CACHE_MISSES("cache-misses"),
086    BRANCHES("branches"),
087    BRANCH_MISSES("branch-misses"),
088    BUS_CYCLES("bus-cycles"),
089    L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"),
090    LLC_LOAD_MISSES("LLC-load-misses"),
091    DTLB_LOAD_MISSES("dTLB-load-misses"),
092    MEM_BREAKPOINT("mem:breakpoint"),
093    TRACE_TRACEPOINT("trace:tracepoint"),;
094
095    private final String internalName;
096
097    Event(final String internalName) {
098      this.internalName = internalName;
099    }
100
101    public String getInternalName() {
102      return internalName;
103    }
104
105    public static Event fromInternalName(final String name) {
106      for (Event event : values()) {
107        if (event.getInternalName().equalsIgnoreCase(name)) {
108          return event;
109        }
110      }
111
112      return null;
113    }
114  }
115
116  enum Output {
117    SUMMARY,
118    TRACES,
119    FLAT,
120    COLLAPSED,
121    // No SVG in 2.x asyncprofiler.
122    SVG,
123    TREE,
124    JFR,
125    // In 2.x asyncprofiler, this is how you get flamegraphs.
126    HTML
127  }
128
129  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED",
130      justification = "This class is never serialized nor restored.")
131  private transient Lock profilerLock = new ReentrantLock();
132  private transient volatile Process process;
133  private String asyncProfilerHome;
134  private Integer pid;
135
136  public ProfileServlet() {
137    this.asyncProfilerHome = getAsyncProfilerHome();
138    this.pid = ProcessUtils.getPid();
139    LOG.info("Servlet process PID: " + pid + " asyncProfilerHome: " + asyncProfilerHome);
140  }
141
142  @Override
143  protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
144    throws IOException {
145    if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), req, resp)) {
146      resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
147      setResponseHeader(resp);
148      resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!");
149      return;
150    }
151
152    // make sure async profiler home is set
153    if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
154      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
155      setResponseHeader(resp);
156      resp.getWriter()
157        .write("ASYNC_PROFILER_HOME env is not set.\n\n"
158          + "Please ensure the prerequsites for the Profiler Servlet have been installed and the\n"
159          + "environment is properly configured. For more information please see\n"
160          + "http://hbase.apache.org/book.html#profiler\n");
161      return;
162    }
163
164    // if pid is explicitly specified, use it else default to current process
165    pid = getInteger(req, "pid", pid);
166
167    // if pid is not specified in query param and if current process pid cannot be determined
168    if (pid == null) {
169      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
170      setResponseHeader(resp);
171      resp.getWriter()
172        .write("'pid' query parameter unspecified or unable to determine PID of current process.");
173      return;
174    }
175
176    final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS);
177    final Output output = getOutput(req);
178    final Event event = getEvent(req);
179    final Long interval = getLong(req, "interval");
180    final Integer jstackDepth = getInteger(req, "jstackdepth", null);
181    final Long bufsize = getLong(req, "bufsize");
182    final boolean thread = req.getParameterMap().containsKey("thread");
183    final boolean simple = req.getParameterMap().containsKey("simple");
184    final Integer width = getInteger(req, "width", null);
185    final Integer height = getInteger(req, "height", null);
186    final Double minwidth = getMinWidth(req);
187    final boolean reverse = req.getParameterMap().containsKey("reverse");
188
189    if (process == null || !process.isAlive()) {
190      try {
191        int lockTimeoutSecs = 3;
192        if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) {
193          try {
194            File outputFile =
195              new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" + event.name().toLowerCase() + "-"
196                + ID_GEN.incrementAndGet() + "." + output.name().toLowerCase());
197            List<String> cmd = new ArrayList<>();
198            cmd.add(asyncProfilerHome + PROFILER_SCRIPT);
199            cmd.add("-e");
200            cmd.add(event.getInternalName());
201            cmd.add("-d");
202            cmd.add("" + duration);
203            cmd.add("-o");
204            cmd.add(output.name().toLowerCase());
205            cmd.add("-f");
206            cmd.add(outputFile.getAbsolutePath());
207            if (interval != null) {
208              cmd.add("-i");
209              cmd.add(interval.toString());
210            }
211            if (jstackDepth != null) {
212              cmd.add("-j");
213              cmd.add(jstackDepth.toString());
214            }
215            if (bufsize != null) {
216              cmd.add("-b");
217              cmd.add(bufsize.toString());
218            }
219            if (thread) {
220              cmd.add("-t");
221            }
222            if (simple) {
223              cmd.add("-s");
224            }
225            if (width != null) {
226              cmd.add("--width");
227              cmd.add(width.toString());
228            }
229            if (height != null) {
230              cmd.add("--height");
231              cmd.add(height.toString());
232            }
233            if (minwidth != null) {
234              cmd.add("--minwidth");
235              cmd.add(minwidth.toString());
236            }
237            if (reverse) {
238              cmd.add("--reverse");
239            }
240            cmd.add(pid.toString());
241            process = ProcessUtils.runCmdAsync(cmd);
242
243            // set response and set refresh header to output location
244            setResponseHeader(resp);
245            resp.setStatus(HttpServletResponse.SC_ACCEPTED);
246            String relativeUrl = "/prof-output-hbase/" + outputFile.getName();
247            resp.getWriter()
248              .write("Started [" + event.getInternalName()
249                + "] profiling. This page will automatically redirect to " + relativeUrl + " after "
250                + duration + " seconds.\n\nCommand:\n" + Joiner.on(" ").join(cmd));
251
252            // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified
253            // via url param
254            int refreshDelay = getInteger(req, "refreshDelay", 0);
255
256            // instead of sending redirect, set auto-refresh so that browsers will refresh
257            // with redirected url
258            resp.setHeader("Refresh", (duration + refreshDelay) + ";" + relativeUrl);
259            resp.getWriter().flush();
260          } finally {
261            profilerLock.unlock();
262          }
263        } else {
264          setResponseHeader(resp);
265          resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
266          resp.getWriter()
267            .write("Unable to acquire lock. Another instance of profiler might be running.");
268          LOG.warn("Unable to acquire lock in " + lockTimeoutSecs
269            + " seconds. Another instance of profiler might be running.");
270        }
271      } catch (InterruptedException e) {
272        LOG.warn("Interrupted while acquiring profile lock.", e);
273        resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
274      }
275    } else {
276      setResponseHeader(resp);
277      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
278      resp.getWriter().write("Another instance of profiler is already running.");
279    }
280  }
281
282  private Integer getInteger(final HttpServletRequest req, final String param,
283    final Integer defaultValue) {
284    final String value = req.getParameter(param);
285    if (value != null) {
286      try {
287        return Integer.valueOf(value);
288      } catch (NumberFormatException e) {
289        return defaultValue;
290      }
291    }
292    return defaultValue;
293  }
294
295  private Long getLong(final HttpServletRequest req, final String param) {
296    final String value = req.getParameter(param);
297    if (value != null) {
298      try {
299        return Long.valueOf(value);
300      } catch (NumberFormatException e) {
301        return null;
302      }
303    }
304    return null;
305  }
306
307  private Double getMinWidth(final HttpServletRequest req) {
308    final String value = req.getParameter("minwidth");
309    if (value != null) {
310      try {
311        return Double.valueOf(value);
312      } catch (NumberFormatException e) {
313        return null;
314      }
315    }
316    return null;
317  }
318
319  private Event getEvent(final HttpServletRequest req) {
320    final String eventArg = req.getParameter("event");
321    if (eventArg != null) {
322      Event event = Event.fromInternalName(eventArg);
323      return event == null ? Event.CPU : event;
324    }
325    return Event.CPU;
326  }
327
328  private Output getOutput(final HttpServletRequest req) {
329    final String outputArg = req.getParameter("output");
330    if (req.getParameter("output") != null) {
331      try {
332        return Output.valueOf(outputArg.trim().toUpperCase());
333      } catch (IllegalArgumentException e) {
334        return Output.HTML;
335      }
336    }
337    return Output.HTML;
338  }
339
340  static void setResponseHeader(final HttpServletResponse response) {
341    response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS);
342    response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
343    response.setContentType(CONTENT_TYPE_TEXT);
344  }
345
346  static String getAsyncProfilerHome() {
347    String asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV);
348    // if ENV is not set, see if -Dasync.profiler.home=/path/to/async/profiler/home is set
349    if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
350      asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY);
351    }
352
353    return asyncProfilerHome;
354  }
355
356  public static class DisabledServlet extends HttpServlet {
357
358    private static final long serialVersionUID = 1L;
359
360    @Override
361    protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
362      throws IOException {
363      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
364      setResponseHeader(resp);
365      resp.getWriter()
366        .write("The profiler servlet was disabled at startup.\n\n"
367          + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n"
368          + "environment is properly configured. For more information please see\n"
369          + "http://hbase.apache.org/book.html#profiler\n");
370      return;
371    }
372
373  }
374
375}