diff --git a/dev-packages/e2e-tests/publish-packages.ts b/dev-packages/e2e-tests/publish-packages.ts
index d342470bd06d..5ade5b1d735c 100644
--- a/dev-packages/e2e-tests/publish-packages.ts
+++ b/dev-packages/e2e-tests/publish-packages.ts
@@ -1,15 +1,26 @@
 import * as childProcess from 'child_process';
+import { readFileSync } from 'fs';
 import * as glob from 'glob';
 import * as path from 'path';
 
 const repositoryRoot = path.resolve(__dirname, '../..');
 
+const version = (JSON.parse(readFileSync(path.join(__dirname, './package.json'), 'utf8')) as { version: string })
+  .version;
+
 // Get absolute paths of all the packages we want to publish to the fake registry
-const packageTarballPaths = glob.sync('packages/*/sentry-*.tgz', {
+// Only include the current versions, to avoid getting old tarballs published as well
+const packageTarballPaths = glob.sync(`packages/*/sentry-*-${version}.tgz`, {
   cwd: repositoryRoot,
   absolute: true,
 });
 
+if (packageTarballPaths.length === 0) {
+  // eslint-disable-next-line no-console
+  console.log(`No packages to publish for version ${version}, did you run "yarn build:tarballs"?`);
+  process.exit(1);
+}
+
 // Publish built packages to the fake registry
 packageTarballPaths.forEach(tarballPath => {
   // eslint-disable-next-line no-console