diff --git a/api/src/org/labkey/api/security/AuthFilter.java b/api/src/org/labkey/api/security/AuthFilter.java index d1e23cbc97f..ae5a8b55373 100644 --- a/api/src/org/labkey/api/security/AuthFilter.java +++ b/api/src/org/labkey/api/security/AuthFilter.java @@ -40,14 +40,16 @@ import org.labkey.api.util.HttpUtil; import org.labkey.api.util.HttpsUtil; import org.labkey.api.util.Pair; +import org.labkey.api.util.Rate; +import org.labkey.api.util.RateLimiter; import org.labkey.api.view.UnauthorizedException; import org.labkey.api.view.ViewServlet; -import org.labkey.api.view.template.PageConfig; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URL; import java.util.Random; +import java.util.concurrent.TimeUnit; @SuppressWarnings({"UnusedDeclaration"}) @@ -237,21 +239,9 @@ else if (!AppProps.getInstance().isDevMode()) QueryService.get().setEnvironment(QueryService.Environment.USER, user); - if (AppProps.getInstance().isOptionalFeatureEnabled("experimental-unsafe-inline")) - { - String csp = StringUtils.trimToEmpty(((HttpServletResponse) response).getHeader("Content-Security-Policy")); - String nonceDirectiveValue = "'nonce-" + PageConfig.getScriptNonceHeader(req) + "'"; - if (!csp.contains(nonceDirectiveValue)) - { - if (!csp.contains("script-src ")) - { - if (StringUtils.isNotBlank(csp)) - csp = StringUtils.appendIfMissing(csp, ";"); - csp += "script-src 'unsafe-eval' http: https: " + nonceDirectiveValue + ";"; - } - ((HttpServletResponse) response).setHeader("Content-Security-Policy", csp); - } - } + // checkRateLimiterOverage() will set response status + if (user.isGuest() && checkRateLimiterOverage((AuthenticatedRequest)req, resp)) + return; try { @@ -322,4 +312,26 @@ private void ensureFirstRequestHandled(HttpServletRequest request) _firstRequestHandled = true; } } + + static final RateLimiter robotLimiter = new RateLimiter("Robots", new Rate(20, TimeUnit.MINUTES)); + + static boolean checkRateLimiterOverage(AuthenticatedRequest req, HttpServletResponse res) + { + var userAgent = req.getHeader("User-Agent"); + /* NOTE: the health checker is currently considered a robot. Don't 429 the healthchecker */ + if (StringUtils.contains(userAgent, "healthchecker")) + return false; + if (!req.isRobot()) + return false; + var count = robotLimiter.getCount(); + var delay = robotLimiter.getDelay(); + if (count >= 10 && delay > 0) + { + res.addHeader("Retry-After", String.valueOf((int)(delay/1000 + 5))); + res.setStatus(429); // TOO MANY REQUESTS why no HttpServletResoonse constant? + return true; + } + robotLimiter.add(1, false); + return false; + } } diff --git a/api/src/org/labkey/api/security/AuthenticatedRequest.java b/api/src/org/labkey/api/security/AuthenticatedRequest.java index 744c3cd64f3..6935f0f1696 100644 --- a/api/src/org/labkey/api/security/AuthenticatedRequest.java +++ b/api/src/org/labkey/api/security/AuthenticatedRequest.java @@ -231,12 +231,15 @@ private boolean isGuest() return _user.isGuest() && !_loggedIn; } - private boolean isRobot() + private Boolean _robot = null; + + public boolean isRobot() { - return PageFlowUtil.isRobotUserAgent(getHeader("User-Agent")); + if (null == _robot) + _robot = PageFlowUtil.isRobotUserAgent(getHeader("User-Agent")); + return _robot; } - // Methods below use reflection to pull Tomcat-specific implementation bits out of the request. This can be helpful // for low-level, temporary debugging, but it's not portable across servlet containers or versions. diff --git a/api/src/org/labkey/api/util/RateAccumulator.java b/api/src/org/labkey/api/util/RateAccumulator.java index 2832482d7b0..52316045120 100644 --- a/api/src/org/labkey/api/util/RateAccumulator.java +++ b/api/src/org/labkey/api/util/RateAccumulator.java @@ -42,6 +42,8 @@ public C getCounter() return _counter; } + // Even with the max(1000) this can be a little aggressive when the time period is short. + // For now the caller can check getCount() if this is a concern. double getRate(long now) { return (double)getCount() / max(1000, now - getStart()); diff --git a/api/src/org/labkey/api/util/RateLimiter.java b/api/src/org/labkey/api/util/RateLimiter.java index 19d6bc094f0..bdd20fa1243 100644 --- a/api/src/org/labkey/api/util/RateLimiter.java +++ b/api/src/org/labkey/api/util/RateLimiter.java @@ -32,8 +32,6 @@ public class RateLimiter { - static final Logger _log = LogManager.getLogger(RateLimiter.class); - final String _name; final Rate _target; boolean useSystem = false; // for small intervals or testing @@ -140,11 +138,13 @@ private long _pause(long delay) } - private synchronized long getDelay() + public synchronized long getDelay() { return _long.getDelay(currentTimeMillis(), _target); } + public synchronized long getCount() { return _long.getCount(); } + private synchronized long _updateCounts(long count) { diff --git a/api/src/org/labkey/api/view/PopupMenuView.java b/api/src/org/labkey/api/view/PopupMenuView.java index 63f69055611..347ca68d243 100644 --- a/api/src/org/labkey/api/view/PopupMenuView.java +++ b/api/src/org/labkey/api/view/PopupMenuView.java @@ -16,11 +16,14 @@ package org.labkey.api.view; +import org.apache.commons.lang3.StringUtils; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.URLHelper; import java.io.IOException; import java.io.PrintWriter; import java.io.Writer; +import java.net.URISyntaxException; public class PopupMenuView extends HttpView { @@ -184,14 +187,45 @@ protected static void renderLink(NavTree item, String cls, Writer out) throws IO if (item.isEmphasis()) styleStr += "font-style: italic;"; + // NOTE: nofollow is not recommended as way to avoid crawling internal links + // instead let's use an onclick handler to "hide" the link + String dataQuery = null; + String href = item.getHref(); + if (null != href && null == item.getScript() && !item.isPost()) + { + try + { + var context = HttpView.currentContext(); + URLHelper url = new URLHelper(href); + if (null != context && context.isRobot()) + url.addParameter("_noindex", "1"); + dataQuery = StringUtils.trimToEmpty(url.getRawQuery()); + if (!dataQuery.isEmpty()) + dataQuery = "?" + dataQuery; + url.deleteParameters(); + href = url.toString(); + HttpView.currentPageConfig().addHandlerForQuerySelector( + "A.noFollowNavigate", + "click", + "window.location = this.href + this.dataset['query']; return false;"); + cls = StringUtils.trimToEmpty(cls) + " noFollowNavigate"; + } + catch (URISyntaxException e) + { + // fall through + } + } + String id = config.makeId("popupMenuView"); out.write(" diff --git a/wiki/src/org/labkey/wiki/WikiController.java b/wiki/src/org/labkey/wiki/WikiController.java index 6de6c06eb89..5520c8ff550 100644 --- a/wiki/src/org/labkey/wiki/WikiController.java +++ b/wiki/src/org/labkey/wiki/WikiController.java @@ -118,6 +118,8 @@ import java.util.TreeSet; import java.util.stream.Collectors; +import static org.labkey.api.data.DataRegion.LAST_FILTER_PARAM; + public class WikiController extends SpringActionController { private static final Logger LOG = LogManager.getLogger(WikiController.class); @@ -1122,6 +1124,14 @@ public PageAction(ViewContext ctx, Wiki wiki, WikiVersion wikiversion) @Override public ModelAndView getView(WikiNameForm form, BindException errors) { + // Don't index page with non default parameters (e.g. targeting webparts in the page) + for (var e = getViewContext().getRequest().getParameterNames() ; e.hasMoreElements() ; ) + { + String p = e.nextElement(); + if (p.contains(".") && !LAST_FILTER_PARAM.equals(p)) + getPageConfig().setNoIndex(); + } + String name = null != form.getName() ? form.getName().trim() : null; //if there's no name parameter, find default page and reload with parameter. //default page is not necessarily same page displayed in wiki web part