diff --git a/tasks/aws-s3-builds-page/createS3Client.js b/tasks/aws-s3-builds-page/createS3Client.js
new file mode 100644
index 00000000..4ff79cde
--- /dev/null
+++ b/tasks/aws-s3-builds-page/createS3Client.js
@@ -0,0 +1,23 @@
+const { S3Client } = require('@aws-sdk/client-s3');
+
+function requiredEnvVar(name) {
+  if (!process.env[name]) {
+    throw new Error(`Environment variable "${name}" is required.`);
+  }
+  return process.env[name];
+}
+
+function createS3Client() {
+  // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
+  requiredEnvVar('AWS_ACCESS_KEY_ID');
+  requiredEnvVar('AWS_SECRET_ACCESS_KEY');
+
+  return {
+    bucket: requiredEnvVar('S3_BUCKET_NAME'),
+    s3Client: new S3Client({
+      region: 'us-east-1'
+    })
+  };
+}
+
+module.exports = { createS3Client };
diff --git a/tasks/aws-s3-builds-page/deleteFile.js b/tasks/aws-s3-builds-page/deleteFile.js
new file mode 100644
index 00000000..8207aefc
--- /dev/null
+++ b/tasks/aws-s3-builds-page/deleteFile.js
@@ -0,0 +1,13 @@
+const { createS3Client } = require('./createS3Client');
+const { DeleteObjectCommand } = require('@aws-sdk/client-s3');
+
+async function deleteFile(name) {
+  const { s3Client, bucket } = createS3Client();
+  const command = new DeleteObjectCommand({
+    Bucket: bucket,
+    Key: name
+  });
+  await s3Client.send(command);
+}
+
+module.exports = { deleteFile };
diff --git a/tasks/aws-s3-builds-page/index.js b/tasks/aws-s3-builds-page/index.js
index 276efc79..a18ffb0a 100644
--- a/tasks/aws-s3-builds-page/index.js
+++ b/tasks/aws-s3-builds-page/index.js
@@ -1,3 +1,5 @@
-const { listBucketFiles } = require('./list-bucket-files');
+const { listFiles } = require('./listFiles');
+const { uploadFile } = require('./uploadFile');
+const { deleteFile } = require('./deleteFile');
 
-module.exports = { listBucketFiles };
+module.exports = { listFiles, uploadFile, deleteFile };
diff --git a/tasks/aws-s3-builds-page/list-bucket-files.js b/tasks/aws-s3-builds-page/list-bucket-files.js
deleted file mode 100644
index c04c87cb..00000000
--- a/tasks/aws-s3-builds-page/list-bucket-files.js
+++ /dev/null
@@ -1,35 +0,0 @@
-const { S3Client, ListObjectsV2Command } = require('@aws-sdk/client-s3');
-
-const bucket = process.env.S3_BUCKET_NAME;
-
-if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY) {
-  // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
-  throw new Error(
-    'AWS credentials must be present in environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY'
-  );
-}
-
-const client = new S3Client({
-  region: 'us-east-1'
-});
-
-async function listBucketFiles() {
-  const command = new ListObjectsV2Command({
-    Bucket: bucket
-  });
-
-  let isTruncated = true;
-  const files = [];
-
-  while (isTruncated) {
-    const { Contents, IsTruncated, NextContinuationToken } = await client.send(
-      command
-    );
-    files.push(...Contents.map(s3obj => s3obj.Key));
-    isTruncated = IsTruncated;
-    command.input.ContinuationToken = NextContinuationToken;
-  }
-  return files
-}
-
-module.exports = { listBucketFiles}
diff --git a/tasks/aws-s3-builds-page/listFiles.js b/tasks/aws-s3-builds-page/listFiles.js
new file mode 100644
index 00000000..55fd248d
--- /dev/null
+++ b/tasks/aws-s3-builds-page/listFiles.js
@@ -0,0 +1,26 @@
+const { ListObjectsV2Command } = require('@aws-sdk/client-s3');
+const { createS3Client } = require('./createS3Client');
+
+async function listFiles() {
+  const { s3Client, bucket } = createS3Client();
+  const command = new ListObjectsV2Command({
+    Bucket: bucket
+  });
+
+  let isTruncated = true;
+  const files = [];
+
+  while (isTruncated) {
+    const {
+      Contents,
+      IsTruncated,
+      NextContinuationToken
+    } = await s3Client.send(command);
+    files.push(...Contents.map(s3obj => s3obj.Key));
+    isTruncated = IsTruncated;
+    command.input.ContinuationToken = NextContinuationToken;
+  }
+  return files;
+}
+
+module.exports = { listFiles };
diff --git a/tasks/aws-s3-builds-page/s3-test.js b/tasks/aws-s3-builds-page/s3-test.js
new file mode 100644
index 00000000..af72d2e3
--- /dev/null
+++ b/tasks/aws-s3-builds-page/s3-test.js
@@ -0,0 +1,65 @@
+/* eslint-disable no-console */
+const { listFiles, uploadFile, deleteFile } = require('./index');
+
+const crypto = require('crypto');
+const uuid = crypto.randomUUID();
+
+const BUCKET_BASE_URL = `https://s3.amazonaws.com/${process.env.S3_BUCKET_NAME}`;
+
+async function run() {
+  const filename = `test-file-${uuid}`;
+  console.log(`Starting test with target file "${filename}"`);
+
+  console.log(`Uploading "${filename}"`);
+  await uploadFile('package.json', filename);
+
+  console.log(`Check if uploaded "${filename}"`);
+  const listing = await listFiles();
+  if (!listing.includes(filename)) {
+    throw new Error(`File "${filename}" has not been uploaded`);
+  }
+
+  console.log(`Check contents of "${filename}"`);
+  const uploadedContents = await (
+    await fetch(BUCKET_BASE_URL + '/' + filename)
+  ).text();
+  expectStringContains('"name": "handlebars"', uploadedContents);
+
+  console.log(`Delete "${filename}"`);
+  await deleteFile(filename);
+
+  console.log(`Check if deleted "${filename}"`);
+  const listingAfterDelete = await listFiles();
+  if (listingAfterDelete.includes(filename)) {
+    throw new Error(`File "${filename}" has not been deleted`);
+  }
+}
+
+run()
+  .finally(logTestFilesInBucket)
+  .catch(error => {
+    console.error(error);
+    process.exit(1);
+  });
+
+function expectStringContains(needle, haystack) {
+  if (!haystack.includes(needle)) {
+    throw new Error(`Expecting to find "${needle}" in string "${haystack}"`);
+  }
+}
+
+async function logTestFilesInBucket() {
+  const listing = await listFiles();
+  const testFilesInBucket = listing.filter(name =>
+    name.startsWith('test-file-')
+  );
+  for (const filename of testFilesInBucket) {
+    console.log(`Detected surplus file "${filename}"`);
+    if (process.argv[2] === '--delete-surplus') {
+      await deleteFile(filename);
+    } else {
+      console.log(`run with --delete-surplus to delete it`);
+    }
+  }
+  console.log('DONE');
+}
diff --git a/tasks/aws-s3-builds-page/test.js b/tasks/aws-s3-builds-page/test.js
deleted file mode 100644
index ec258084..00000000
--- a/tasks/aws-s3-builds-page/test.js
+++ /dev/null
@@ -1,7 +0,0 @@
-const { listBucketFiles } = require('./index');
-
-const main = async () => {
-  console.log(await listBucketFiles());
-};
-
-main().catch(console.error);
diff --git a/tasks/aws-s3-builds-page/uploadFile.js b/tasks/aws-s3-builds-page/uploadFile.js
new file mode 100644
index 00000000..00095956
--- /dev/null
+++ b/tasks/aws-s3-builds-page/uploadFile.js
@@ -0,0 +1,16 @@
+const { createS3Client } = require('./createS3Client');
+const { PutObjectCommand } = require('@aws-sdk/client-s3');
+const fs = require('node:fs/promises');
+
+async function uploadFile(localName, targetName) {
+  const { s3Client, bucket } = createS3Client();
+  const fileContents = await fs.readFile(localName);
+  const command = new PutObjectCommand({
+    Bucket: bucket,
+    Key: targetName,
+    Body: fileContents
+  });
+  await s3Client.send(command);
+}
+
+module.exports = { uploadFile };
diff --git a/tests/integration/webpack-babel-test/webpack.config.js b/tests/integration/webpack-babel-test/webpack.config.js
index bebb6ee0..50553f21 100644
--- a/tests/integration/webpack-babel-test/webpack.config.js
+++ b/tests/integration/webpack-babel-test/webpack.config.js
@@ -3,7 +3,7 @@ const fs = require('fs');
 const testFiles = fs.readdirSync('src');
 const entryPoints = {};
 testFiles
-  .filter(file => file.match(/-test.js$/))
+  .filter(file => file.match(/-cli.js$/))
   .forEach(file => {
     entryPoints[file] = `./src/${file}`;
   });
diff --git a/tests/integration/webpack-test/webpack.config.js b/tests/integration/webpack-test/webpack.config.js
index 207aed30..720e5628 100644
--- a/tests/integration/webpack-test/webpack.config.js
+++ b/tests/integration/webpack-test/webpack.config.js
@@ -3,7 +3,7 @@ const fs = require('fs');
 const testFiles = fs.readdirSync('src');
 const entryPoints = {};
 testFiles
-  .filter(file => file.match(/-test.js$/))
+  .filter(file => file.match(/-cli.js$/))
   .forEach(file => {
     entryPoints[file] = `./src/${file}`;
   });