diff --git a/.gitignore b/.gitignore index c3dac9ba5..e5cb4b4f2 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ META-INF/ /.apt_generated/ /.apt_generated_tests/ /bin/ +/.asciidoctorconfig.adoc diff --git a/docs/USER_GUIDE.adoc b/docs/USER_GUIDE.adoc index db5e43984..ac9ff0ee3 100644 --- a/docs/USER_GUIDE.adoc +++ b/docs/USER_GUIDE.adoc @@ -94,8 +94,8 @@ You can setup a custom Jenkins URL to be used as callback URL by the webhooks. For Bitbucket Data Center only, it is possible chose which webhooks implementation server side to use: - Native implementation will configure the webhooks provided by default with the Server, so it will always be available. - -- Plugin implementation (*deprecated*) relies on the configuration available via specific APIs provided by the link:https://marketplace.atlassian.com/apps/1215474/post-webhooks-for-bitbucket?tab=overview&hosting=datacenter[Post Webhooks for Bitbucket] plugin itself. To get it worked plugin must be already pre-installed on the server instance. This provider allows custom settings managed by the _ignore committers_ trait. _Note: This specific implementation will be moved to an individual repository as soon as link:https://issues.jenkins.io/browse/JENKINS-74913[JENKINS-74913] is implemented._ +- Plugin implementation (*deprecated*) replaced by https://github.com/jenkinsci/bitbucket-webhooks-plugin[this plugin] +- Any other extension point implementation installed in your Jenkins instance image::images/screenshot-14.png[] @@ -120,6 +120,35 @@ Any incoming webhook payloads from that given endpoint will be validated against image::images/screenshot-20.png[] +=== Enable webhooks cache + +If your organisation has a large number of repositories (over 500), you may easily reach the API rate limit. +Any requests made to Bitbucket beyond 1,000 per hour will be rejected. In this case, enable caching to allow webhooks from repositories beyond the 500th to be processed within the next hour (when the rate is unlocked). + +See https://issues.jenkins.io/browse/JENKINS-76184[JENKINS-76184] for the use case. + +image::images/screenshot-23.png[] + +=== Manual registration + +If your organisation does not allow credentials to handle repository webhooks than you can provide to register webhook manually. You can follow one of these official Atlassian guides: for https://support.atlassian.com/bitbucket-cloud/docs/manage-webhooks[Cloud] or for https://confluence.atlassian.com/bitbucketserver/manage-webhooks-938025878.html[Data Center]. + +Go to the Bitbucket _Repository_ » _Repository settings_ » _Webhooks than _Add Webhook_ + +Provide a title of your choice, if you generate a secret for payload verification than confiure signature verification in Bitbucket endpoint as in the previous chapter. + +Select events as shown in the image; any other types selected will be ignored. + +The callback URL must be configured as follow: +* Cloud: /bitbucket-scmsource-hook/notify +* Server: /bitbucket-scmsource-hook/notify?server_url= + +The must match the Jenkins public host; if Jenkins is behind a reverse proxy, ensure the URL provided matches the one in Manage Jenkins » System » Jenkins Location. +In other cases, you can provide the Jenkins root URL to use, in the webhook configuration page on the Bitbucket endpoint. +The must match the server address configured in the Bitbucket endpoint. Otherwise, incoming webhooks will be discarded. + +image::images/screenshot-24.png[] + [id=bitbucket-creds-config] == Credentials configuration diff --git a/docs/images/screenshot-23.png b/docs/images/screenshot-23.png new file mode 100644 index 000000000..eab1abc8b Binary files /dev/null and b/docs/images/screenshot-23.png differ diff --git a/docs/images/screenshot-24.png b/docs/images/screenshot-24.png new file mode 100644 index 000000000..76783e554 Binary files /dev/null and b/docs/images/screenshot-24.png differ diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java index ab80b71c8..0b6878cf8 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java @@ -113,7 +113,7 @@ public class BitbucketCloudApiClient extends AbstractBitbucketApi implements Bit private static final Cache cachedTeam = new Cache<>(6, HOURS); private static final Cache> cachedRepositories = new Cache<>(3, HOURS); private static final Cache cachedCommits = new Cache<>(24, HOURS); - private transient BitbucketRepository cachedRepository; + private transient BitbucketRepository localCachedRepository; private transient String cachedDefaultBranch; public static List stats() { @@ -265,14 +265,14 @@ public BitbucketRepository getRepository() throws IOException { if (repositoryName == null) { throw new UnsupportedOperationException("Cannot get a repository from an API instance that is not associated with a repository"); } - if (!enableCache || cachedRepository == null) { + if (!enableCache || localCachedRepository == null) { String url = UriTemplate.fromTemplate(REPO_URL_TEMPLATE) .set("owner", owner) .set("repo", repositoryName) .expand(); - cachedRepository = getRequestAs(url, BitbucketCloudRepository.class); + localCachedRepository = getRequestAs(url, BitbucketCloudRepository.class); } - return cachedRepository; + return localCachedRepository; } /** diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiFactory.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiFactory.java index 45cf2c8f0..54657051f 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiFactory.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiFactory.java @@ -48,8 +48,8 @@ protected BitbucketApi create(@Nullable String serverUrl, @Nullable BitbucketAut .lookupEndpoint(BitbucketCloudEndpoint.SERVER_URL, BitbucketCloudEndpoint.class) .orElse(null); boolean enableCache = false; - int teamCacheDuration = 0; - int repositoriesCacheDuration = 0; + int teamCacheDuration = 360; + int repositoriesCacheDuration = 180; if (endpoint != null) { enableCache = endpoint.isEnableCache(); teamCacheDuration = endpoint.getTeamCacheDuration(); diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration.java index 920c3815a..923427ba1 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration.java @@ -39,6 +39,7 @@ import hudson.util.ListBoxModel; import java.net.MalformedURLException; import java.net.URL; +import java.util.List; import jenkins.model.Jenkins; import org.apache.commons.lang3.StringUtils; import org.jenkinsci.plugins.plaincredentials.StringCredentials; @@ -84,6 +85,13 @@ public abstract class AbstractBitbucketWebhookConfiguration implements Bitbucket */ private String endpointJenkinsRootURL; + private boolean enableCache = false; + + /** + * How long, in minutes, to cache the webhook response. + */ + private Integer webhooksCacheDuration; + protected AbstractBitbucketWebhookConfiguration(boolean manageHooks, @CheckForNull String credentialsId, boolean enableHookSignature, @CheckForNull String hookSignatureCredentialsId) { this.manageHooks = manageHooks && StringUtils.isNotBlank(credentialsId); @@ -144,7 +152,47 @@ public String getDisplayName() { return Messages.ServerWebhookImplementation_displayName(); } + public boolean isEnableCache() { + return enableCache; + } + + @DataBoundSetter + public void setEnableCache(boolean enableCache) { + this.enableCache = enableCache; + } + + public Integer getWebhooksCacheDuration() { + return webhooksCacheDuration; + } + + @DataBoundSetter + public void setWebhooksCacheDuration(Integer webhooksCacheDuration) { + this.webhooksCacheDuration = webhooksCacheDuration == null || webhooksCacheDuration < 0 ? Integer.valueOf(180) : webhooksCacheDuration; + } + public abstract static class AbstractBitbucketWebhookDescriptorImpl extends BitbucketWebhookDescriptor { + protected abstract void clearCaches(); + protected abstract List getStats(); + + @RequirePOST + public FormValidation doShowStats() { + Jenkins.get().checkPermission(Jenkins.MANAGE); + + List stats = getStats(); + StringBuilder builder = new StringBuilder(); + for (String stat : stats) { + builder.append(stat).append("
"); + } + return FormValidation.okWithMarkup(builder.toString()); + } + + @RequirePOST + public FormValidation doClear() { + Jenkins.get().checkPermission(Jenkins.MANAGE); + + clearCaches(); + return FormValidation.ok("Caches cleared"); + } @RequirePOST public static FormValidation doCheckEndpointJenkinsRootURL(@QueryParameter String value) { diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookConfiguration.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookConfiguration.java index 31748fc60..5967447a0 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookConfiguration.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookConfiguration.java @@ -32,6 +32,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import hudson.Extension; import hudson.util.ListBoxModel; +import java.util.List; import jenkins.model.Jenkins; import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; @@ -69,6 +70,16 @@ public Class getManager() { @Extension public static class DescriptorImpl extends AbstractBitbucketWebhookDescriptorImpl { + @Override + protected void clearCaches() { + CloudWebhookManager.clearCaches(); + } + + @Override + protected List getStats() { + return CloudWebhookManager.stats(); + } + @Override public String getDisplayName() { return "Native Cloud"; diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookManager.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookManager.java index 7501793e0..d665c7887 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookManager.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookManager.java @@ -24,14 +24,18 @@ package com.cloudbees.jenkins.plugins.bitbucket.impl.webhook.cloud; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketAuthenticatedClient; +import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRequestException; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketWebHook; import com.cloudbees.jenkins.plugins.bitbucket.api.endpoint.BitbucketEndpoint; import com.cloudbees.jenkins.plugins.bitbucket.api.webhook.BitbucketWebhookConfiguration; import com.cloudbees.jenkins.plugins.bitbucket.api.webhook.BitbucketWebhookManager; import com.cloudbees.jenkins.plugins.bitbucket.client.BitbucketCloudPage; +import com.cloudbees.jenkins.plugins.bitbucket.client.Cache; import com.cloudbees.jenkins.plugins.bitbucket.client.repository.BitbucketCloudWebhook; import com.cloudbees.jenkins.plugins.bitbucket.hooks.HookEventType; +import com.cloudbees.jenkins.plugins.bitbucket.impl.client.ICheckedCallable; import com.cloudbees.jenkins.plugins.bitbucket.impl.endpoint.BitbucketCloudEndpoint; +import com.cloudbees.jenkins.plugins.bitbucket.impl.util.BitbucketApiUtils; import com.cloudbees.jenkins.plugins.bitbucket.impl.util.JsonParser; import com.cloudbees.jenkins.plugins.bitbucket.util.BitbucketCredentialsUtils; import com.damnhandy.uri.template.UriTemplate; @@ -48,6 +52,7 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.logging.Logger; import jenkins.model.Jenkins; @@ -57,10 +62,25 @@ import org.apache.commons.lang3.Strings; import org.jenkinsci.plugins.plaincredentials.StringCredentials; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static org.apache.commons.lang3.StringUtils.upperCase; + @Extension public class CloudWebhookManager implements BitbucketWebhookManager { private static final String WEBHOOK_URL = "/2.0/repositories{/owner,repo}/hooks{/hook}{?page,pagelen}"; private static final Logger logger = Logger.getLogger(CloudWebhookManager.class.getName()); + private static final Cache> cachedRepositoryWebhooks = new Cache<>(3, HOURS); + + public static void clearCaches() { + cachedRepositoryWebhooks.evictAll(); + } + + public static List stats() { + List stats = new ArrayList<>(); + stats.add("Repositories webhooks: " + cachedRepositoryWebhooks.stats().toString()); + return stats; + } // The list of events available in Bitbucket Cloud. private static final List CLOUD_EVENTS = Collections.unmodifiableList(Arrays.asList( @@ -87,6 +107,9 @@ public void apply(SCMSourceTrait configurationTrait) { @Override public void apply(BitbucketWebhookConfiguration configuration) { this.configuration = (CloudWebhookConfiguration) configuration; + if (this.configuration.isEnableCache()) { + cachedRepositoryWebhooks.setExpireDuration(this.configuration.getWebhooksCacheDuration(), MINUTES); + } } @Override @@ -105,21 +128,38 @@ public Collection read(@NonNull BitbucketAuthenticatedClient c .set("pagelen", 100) .expand(); - List resources = new ArrayList<>(); + ICheckedCallable, IOException> request = () -> { + List resources = new ArrayList<>(); - TypeReference> type = new TypeReference>(){}; - BitbucketCloudPage page = JsonParser.toJava(client.get(url), type); - resources.addAll(page.getValues().stream() - .filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL)) - .toList()); - while (!page.isLastPage()){ - String response = client.get(page.getNext()); - page = JsonParser.toJava(response, type); + TypeReference> type = new TypeReference>(){}; + BitbucketCloudPage page = JsonParser.toJava(client.get(url), type); resources.addAll(page.getValues().stream() .filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL)) .toList()); + while (!page.isLastPage()){ + String response = client.get(page.getNext()); + page = JsonParser.toJava(response, type); + resources.addAll(page.getValues().stream() + .filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL)) + .toList()); + } + return resources; + }; + if (configuration.isEnableCache()) { + try { + String cacheKey = upperCase(client.getRepositoryOwner()) + "::" + ObjectUtils.firstNonNull(client.getRepositoryName(), ""); + return cachedRepositoryWebhooks.get(cacheKey, request); + } catch (ExecutionException e) { + BitbucketRequestException bre = BitbucketApiUtils.unwrap(e); + if (bre != null) { + throw bre; + } else { + throw new IOException(e); + } + } + } else { + return request.call(); } - return resources; } @NonNull diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/plugin/PluginPushWebhookProcessor.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/plugin/PluginPushWebhookProcessor.java index 68515b01a..2dba521a0 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/plugin/PluginPushWebhookProcessor.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/plugin/PluginPushWebhookProcessor.java @@ -61,6 +61,8 @@ public boolean canHandle(Map headers, MultiValuedMap context, @NonNull BitbucketEndpoint endpoint) { + logger.warning("Plugin webhook is deprecated, it has been replaced by the bitbucket-webhooks-plugin, documentation available at https://github.com/jenkinsci/bitbucket-webhooks-plugin."); + BitbucketPushEvent push = BitbucketServerWebhookPayload.pushEventFromPayload(payload); if (push != null) { if (push.getChanges().isEmpty()) { diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/server/ServerWebhookConfiguration.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/server/ServerWebhookConfiguration.java index 7350641de..726dde1de 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/server/ServerWebhookConfiguration.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/server/ServerWebhookConfiguration.java @@ -32,6 +32,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import hudson.Extension; import hudson.util.ListBoxModel; +import java.util.List; import jenkins.model.Jenkins; import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; @@ -72,6 +73,16 @@ public Class getManager() { @Extension public static class DescriptorImpl extends AbstractBitbucketWebhookDescriptorImpl { + @Override + protected void clearCaches() { + ServerWebhookManager.clearCaches(); + } + + @Override + protected List getStats() { + return ServerWebhookManager.stats(); + } + @Override public String getDisplayName() { return "Native Data Center"; diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/server/ServerWebhookManager.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/server/ServerWebhookManager.java index ddef141fb..6258dd1c7 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/server/ServerWebhookManager.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/server/ServerWebhookManager.java @@ -24,11 +24,15 @@ package com.cloudbees.jenkins.plugins.bitbucket.impl.webhook.server; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketAuthenticatedClient; +import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRequestException; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketWebHook; import com.cloudbees.jenkins.plugins.bitbucket.api.endpoint.BitbucketEndpoint; import com.cloudbees.jenkins.plugins.bitbucket.api.webhook.BitbucketWebhookConfiguration; import com.cloudbees.jenkins.plugins.bitbucket.api.webhook.BitbucketWebhookManager; +import com.cloudbees.jenkins.plugins.bitbucket.client.Cache; import com.cloudbees.jenkins.plugins.bitbucket.hooks.HookEventType; +import com.cloudbees.jenkins.plugins.bitbucket.impl.client.ICheckedCallable; +import com.cloudbees.jenkins.plugins.bitbucket.impl.util.BitbucketApiUtils; import com.cloudbees.jenkins.plugins.bitbucket.impl.util.JsonParser; import com.cloudbees.jenkins.plugins.bitbucket.server.client.BitbucketServerPage; import com.cloudbees.jenkins.plugins.bitbucket.server.client.repository.BitbucketServerWebhook; @@ -47,6 +51,7 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.logging.Logger; import jenkins.model.Jenkins; @@ -55,10 +60,25 @@ import org.apache.commons.lang3.ObjectUtils; import org.jenkinsci.plugins.plaincredentials.StringCredentials; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static org.apache.commons.lang3.StringUtils.upperCase; + @Extension public class ServerWebhookManager implements BitbucketWebhookManager { private static final String WEBHOOK_API = "/rest/api/1.0/projects/{owner}/repos/{repo}/webhooks{/id}{?start,limit}"; private static final Logger logger = Logger.getLogger(ServerWebhookManager.class.getName()); + private static final Cache> cachedRepositoryWebhooks = new Cache<>(3, HOURS); + + public static void clearCaches() { + cachedRepositoryWebhooks.evictAll(); + } + + public static List stats() { + List stats = new ArrayList<>(); + stats.add("Repositories webhooks: " + cachedRepositoryWebhooks.stats().toString()); + return stats; + } // The list of events available in Bitbucket Data Center for the minimum supported version. private static final List NATIVE_SERVER_EVENTS = Collections.unmodifiableList(Arrays.asList( @@ -99,6 +119,9 @@ public void setCallbackURL(@NonNull String callbackURL, @NonNull BitbucketEndpoi @Override public void apply(BitbucketWebhookConfiguration configuration) { this.configuration = (ServerWebhookConfiguration) configuration; + if (this.configuration.isEnableCache()) { + cachedRepositoryWebhooks.setExpireDuration(this.configuration.getWebhooksCacheDuration(), MINUTES); + } } @Override @@ -113,12 +136,29 @@ public Collection read(@NonNull BitbucketAuthenticatedClient c .set("limit", 200) .expand(); - TypeReference> type = new TypeReference>(){}; - return JsonParser.toJava(client.get(url), type) - .getValues().stream() - .map(BitbucketWebHook.class::cast) - .filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL)) - .toList(); + ICheckedCallable, IOException> request = () -> { + TypeReference> type = new TypeReference>(){}; + return JsonParser.toJava(client.get(url), type) + .getValues().stream() + .map(BitbucketWebHook.class::cast) + .filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL)) + .toList(); + }; + if (configuration.isEnableCache()) { + try { + String cacheKey = upperCase(client.getRepositoryOwner()) + "::" + ObjectUtils.firstNonNull(client.getRepositoryName(), ""); + return cachedRepositoryWebhooks.get(cacheKey, request); + } catch (ExecutionException e) { + BitbucketRequestException bre = BitbucketApiUtils.unwrap(e); + if (bre != null) { + throw bre; + } else { + throw new IOException(e); + } + } + } else { + return request.call(); + } } @NonNull diff --git a/src/main/resources/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration/config.jelly b/src/main/resources/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration/config.jelly index 592f4af80..b1a69f850 100644 --- a/src/main/resources/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration/config.jelly +++ b/src/main/resources/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration/config.jelly @@ -38,4 +38,12 @@ THE SOFTWARE. + + + + + + + +