remove old aws-sdk

generate file list after publish
function to upload a file list
2023-08-06 01:20:58 +02:00 · 2023-08-06 01:17:42 +02:00 · 2023-08-05 18:26:51 +02:00 · 2023-08-05 18:09:54 +02:00 · 2023-08-05 15:42:32 +02:00 · 2023-08-05 15:11:00 +02:00
16 changed files with 4191 additions and 247 deletions
@@ -30,8 +30,8 @@
    "uglify-js": "^3.1.4"
  },
  "devDependencies": {
+    "@aws-sdk/client-s3": "^3.385.0",
    "@playwright/test": "^1.17.1",
-    "aws-sdk": "^2.1.49",
    "babel-loader": "^5.0.0",
    "babel-runtime": "^5.1.10",
    "benchmark": "~1.0",
@@ -0,0 +1,127 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport"
+          content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <style>
+        table {
+            column-gap: 1rem;
+        }
+        thead {
+            position: sticky;
+            top: 0;
+            background: #efefef;
+        }
+
+        th:not(:last-child), td:not(:last-child) {
+            margin-right: 1rem;
+        }
+
+        th {
+            text-align: left;
+        }
+    </style>
+    <title>Handlebars.js Builds</title>
+</head>
+<body>
+ <h1>Handlebars.js builds</h1>
+ <p>See <a href="https://handlebarsjs.com">https://handlebarsjs.com</a> for documentation.</p>
+ <p>Machine-readable version: <a href="{{jsonListUrl}}">{{jsonListUrl}}</a></p>
+ <table>
+     <thead>
+         <tr>
+             <th data-col="key"><a href="#" onclick="return toggleSort('key')">Name</a></th>
+             <th data-col="size"><a href="#" onclick="return toggleSort('size')">Size</a></th>
+             <th data-col="lastModified"><a href="#" onclick="return toggleSort('lastModified')">Last-Modified</a></th>
+         </tr>
+     </thead>
+     <tbody id="files">
+         {{#each fileList as | file |}}
+             <tr>
+                 <td data-col="key"><a href="{{file.key}}">{{key}}</a></td>
+                 <td data-col="size">{{file.size}}</td>
+                 <td data-col="lastModified">{{file.lastModified}}</td>
+             </tr>
+         {{/each}}
+     </tbody>
+
+ </table>
+ <script type="application/javascript">
+     const files = {{{json fileList}}};
+     const fileElements = Array.from(document.querySelectorAll("#files > tr"));
+
+
+     applyNewOrder()
+
+     function getSearchParams() {
+         return new URLSearchParams(window.location.hash.slice(1));
+     }
+
+     function toggleSort(newSortProperty) {
+         const params = getSearchParams()
+
+         const oldSortProperty = params.get("sort");
+         if (oldSortProperty === newSortProperty) {
+             const newDir = params.get("dir") === "asc" ? "desc" : "asc"
+             window.location.hash = "sort=" + newSortProperty + "&dir=" + newDir
+         } else {
+             window.location.hash = "sort=" + newSortProperty
+
+         }
+         setTimeout(() => applyNewOrder())
+
+
+         return false
+     }
+
+     function applyNewOrder() {
+         const params = getSearchParams()
+         const sortProperty = params.get("sort") ?? "lastModified"
+         const ascending = params.get("dir") === "asc"
+         sortFilesArray(sortProperty, ascending);
+         updateRows();
+     }
+
+     function sortFilesArray(propertyName, ascending) {
+
+         files.sort(compareByProp(propertyName))
+         if (!ascending) {
+             files.reverse()
+         }
+     }
+
+     function compareByProp(propertyName) {
+         return (file1, file2) => {
+             if (file1[propertyName] === file2[propertyName]) {
+                 return 0
+             }
+             if (file1[propertyName] > file2[propertyName]) {
+                 return 1
+             }
+             return -1
+         }
+     }
+
+     function updateRows() {
+         let index = 0;
+         for (const rowElement of fileElements) {
+             update(rowElement, files[index++])
+         }
+     }
+
+     function update(rowElement, file) {
+         const link = rowElement.querySelector('[data-col="key"] a')
+         link.setAttribute("href", file.key)
+         link.innerText = file.key
+
+         const size = rowElement.querySelector('[data-col="size"]')
+         size.innerText = file.size
+
+         const lastModified = rowElement.querySelector('[data-col="lastModified"]')
+         lastModified.innerText = file.lastModified
+     }
+ </script>
+</body>
+</html>
@@ -0,0 +1,31 @@
+const crypto = require('crypto');
+
+const { runTest } = require('./test-utils/runTest');
+const { createS3Client } = require('./s3client');
+const { generateFileList } = require('./generateFileList');
+const assert = require('node:assert');
+
+// This is a test file. It is intended to be run manually with the proper environment variables set
+//
+// Run it from the project root using "node tasks/aws-s3-builds-page/generateFileList-test.js"
+
+const s3Client = createS3Client();
+
+runTest(async ({ log }) => {
+  log('Generate file list');
+  const filename = `test-file-list-${crypto.randomUUID()}`;
+  await generateFileList(filename);
+
+  log(`Checking JSON at ${s3Client.fileUrl(`${filename}.json`)}`);
+  const jsonList = JSON.parse(await s3Client.fetchFile(`${filename}.json`));
+  assert(jsonList.find(s3obj => s3obj.key === 'handlebars-v4.7.7.js'));
+
+  log(`Checking HTML at ${s3Client.fileUrl(`${filename}.html`)}`);
+  const htmlList = await s3Client.fetchFile(`${filename}.html`);
+  assert(htmlList.includes('handlebars-v4.7.7.js'));
+  assert(htmlList.includes('handlebarsjs.com'));
+  assert(!htmlList.includes('index.html'));
+
+  log(`Deleting file ${filename}.json`);
+  await s3Client.deleteFile(`${filename}.json`);
+});
@@ -0,0 +1,39 @@
+/* eslint-disable no-console */
+const { createS3Client } = require('./s3client');
+const Handlebars = require('../..');
+const fs = require('node:fs/promises');
+const path = require('path');
+
+async function generateFileList(nameWithoutExtension) {
+  const s3Client = createS3Client();
+  const fileList = await s3Client.listFiles();
+  const relevantFiles = fileList.filter(s3obj => s3obj.key.endsWith('.js'));
+
+  await uploadJson(s3Client, relevantFiles, nameWithoutExtension);
+  await uploadHtml(s3Client, relevantFiles, nameWithoutExtension);
+}
+
+async function uploadJson(s3Client, fileList, nameWithoutExtension) {
+  const fileListJson = JSON.stringify(fileList, null, 2);
+  await s3Client.uploadData(fileListJson, nameWithoutExtension + '.json', {
+    contentType: 'application/json'
+  });
+}
+
+async function uploadHtml(s3Client, fileList, nameWithoutExtension) {
+  const templateStr = await fs.readFile(
+    path.join(__dirname, 'fileList.hbs'),
+    'utf-8'
+  );
+  const template = Handlebars.compile(templateStr);
+  Handlebars.registerHelper('json', obj => JSON.stringify(obj));
+  const fileListHtml = template({
+    fileList,
+    jsonListUrl: nameWithoutExtension + '.json'
+  });
+  await s3Client.uploadData(fileListHtml, nameWithoutExtension + '.html', {
+    contentType: 'text/html'
+  });
+}
+
+module.exports = { generateFileList };
@@ -0,0 +1,47 @@
+const crypto = require('crypto');
+const { publishWithSuffixes } = require('./publish');
+const { runTest } = require('./test-utils/runTest');
+const { createS3Client } = require('./s3client');
+const fs = require('node:fs/promises');
+
+// This is a test file. It is intended to be run manually with the proper environment variables set
+//
+// Run it from the project root using "node tasks/aws-s3-builds-page/publish-test.js"
+
+const s3Client = createS3Client();
+
+runTest(async ({ log }) => {
+  const suffix1 = `-test-file-` + crypto.randomUUID();
+  const suffix2 = `-test-file-` + crypto.randomUUID();
+  log(`Publish ${suffix1} and ${suffix2}`);
+  await publishWithSuffixes([suffix1, suffix2]);
+  await compareAndDeleteFiles(suffix1, log);
+  await compareAndDeleteFiles(suffix2, log);
+});
+
+async function compareAndDeleteFiles(suffix, log) {
+  const pairs = [
+    ['dist/handlebars.js', `handlebars${suffix}.js`],
+    ['dist/handlebars.min.js', `handlebars.min${suffix}.js`],
+    ['dist/handlebars.runtime.js', `handlebars.runtime${suffix}.js`],
+    ['dist/handlebars.runtime.min.js', `handlebars.runtime.min${suffix}.js`]
+  ];
+  for (const [localFile, remoteFile] of pairs) {
+    await expectSameContents(localFile, remoteFile, log);
+    log(`Deleting "${remoteFile}"`);
+    await s3Client.deleteFile(remoteFile);
+  }
+}
+
+async function expectSameContents(localFile, remoteFile, log) {
+  log(
+    `Checking file contents "${localFile}" vs "${s3Client.fileUrl(remoteFile)}"`
+  );
+  const remoteContents = await s3Client.fetchFile(remoteFile);
+  const localContents = await fs.readFile(localFile, 'utf-8');
+  if (remoteContents !== localContents) {
+    throw new Error(
+      `Files do not match: ${localFile}" vs "${s3Client.fileUrl(remoteFile)}"`
+    );
+  }
+}
@@ -0,0 +1,37 @@
+/* eslint-disable no-console */
+const { createS3Client } = require('./s3client');
+
+const filenames = [
+  'handlebars.js',
+  'handlebars.min.js',
+  'handlebars.runtime.js',
+  'handlebars.runtime.min.js'
+];
+
+async function publishWithSuffixes(suffixes) {
+  const s3Client = createS3Client();
+  const publishPromises = suffixes.map(suffix =>
+    publishSuffix(s3Client, suffix)
+  );
+  return Promise.all(publishPromises);
+}
+
+async function publishSuffix(s3client, suffix) {
+  const publishPromises = filenames.map(async filename => {
+    const nameInBucket = getNameInBucket(filename, suffix);
+    const localFile = getLocalFile(filename);
+    await s3client.uploadFile(localFile, nameInBucket);
+    console.log(`Published ${localFile} to build server (${nameInBucket})`);
+  });
+  return Promise.all(publishPromises);
+}
+
+function getNameInBucket(filename, suffix) {
+  return filename.replace(/\.js$/, suffix + '.js');
+}
+
+function getLocalFile(filename) {
+  return 'dist/' + filename;
+}
+
+module.exports = { publishWithSuffixes };
@@ -0,0 +1,11 @@
+const { DeleteObjectCommand } = require('@aws-sdk/client-s3');
+
+async function deleteFile(s3Client, bucket, remoteName) {
+  const command = new DeleteObjectCommand({
+    Bucket: bucket,
+    Key: remoteName
+  });
+  await s3Client.send(command);
+}
+
+module.exports = { deleteFile };
@@ -0,0 +1,10 @@
+async function fetchFile(bucket, remoteName) {
+  return (await fetch(fileUrl(bucket, remoteName))).text();
+}
+
+function fileUrl(bucket, remoteName) {
+  const bucketUrl = `https://s3.amazonaws.com/${bucket}`;
+  return `${bucketUrl}/${remoteName}`;
+}
+
+module.exports = { fetchFile, fileUrl };
@@ -0,0 +1,42 @@
+const { listFiles } = require('./listFiles');
+const { uploadFile, uploadData } = require('./uploadFile');
+const { deleteFile } = require('./deleteFile');
+const { S3Client } = require('@aws-sdk/client-s3');
+const { requireEnvVar } = require('./requireEnvVar');
+const { fetchFile, fileUrl } = require('./fetchFile');
+
+module.exports = { createS3Client };
+
+function createS3Client() {
+  // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
+  requireEnvVar('AWS_ACCESS_KEY_ID');
+  requireEnvVar('AWS_SECRET_ACCESS_KEY');
+
+  const bucket = requireEnvVar('S3_BUCKET_NAME');
+  const s3Client = new S3Client({
+    region: 'us-east-1'
+  });
+
+  return {
+    async listFiles() {
+      return listFiles(s3Client, bucket);
+    },
+    async uploadFile(localName, remoteName, { contentType } = {}) {
+      await uploadFile(s3Client, bucket, localName, remoteName, {
+        contentType
+      });
+    },
+    async uploadData(data, remoteName, { contentType } = {}) {
+      await uploadData(s3Client, bucket, data, remoteName, { contentType });
+    },
+    async deleteFile(remoteName) {
+      await deleteFile(s3Client, bucket, remoteName);
+    },
+    async fetchFile(remoteName) {
+      return fetchFile(bucket, remoteName);
+    },
+    fileUrl(remoteName) {
+      return fileUrl(bucket, remoteName);
+    }
+  };
+}
@@ -0,0 +1,32 @@
+const { ListObjectsV2Command } = require('@aws-sdk/client-s3');
+
+async function listFiles(s3Client, bucket) {
+  const command = new ListObjectsV2Command({
+    Bucket: bucket
+  });
+
+  let isTruncated = true;
+  const files = [];
+
+  while (isTruncated) {
+    const {
+      Contents,
+      IsTruncated,
+      NextContinuationToken
+    } = await s3Client.send(command);
+    files.push(...Contents.map(dataFromS3Object));
+    isTruncated = IsTruncated;
+    command.input.ContinuationToken = NextContinuationToken;
+  }
+  return files;
+}
+
+function dataFromS3Object(s3obj) {
+  return {
+    key: s3obj.Key,
+    size: s3obj.Size,
+    lastModified: s3obj.LastModified.toISOString()
+  };
+}
+
+module.exports = { listFiles };
@@ -0,0 +1,8 @@
+function requireEnvVar(name) {
+  if (!process.env[name]) {
+    throw new Error(`Environment variable "${name}" is required.`);
+  }
+  return process.env[name];
+}
+
+module.exports = { requireEnvVar };
@@ -0,0 +1,89 @@
+/* eslint-disable no-console */
+const { createS3Client } = require('./index');
+const crypto = require('crypto');
+const { runTest } = require('../test-utils/runTest');
+const assert = require('node:assert');
+
+// This is a test file. It is intended to be run manually
+// with the proper environment variables set
+// It tests whether the upload/list/delete methods in this directory
+// work properly.
+//
+// Run it from the project root using "node tasks/aws-s3-builds-page/s3client/s3-test.js"
+
+const client = createS3Client();
+
+const ISO_DATE = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/;
+
+runTest(async ({ log }) => {
+  const uuid = crypto.randomUUID();
+  const filename = `test-file-${uuid}`;
+  log(`Starting test with target file "${filename}"`);
+
+  log(`Uploading "${filename}"`);
+  await client.uploadFile('package.json', filename);
+
+  log(`Check if uploaded "${filename}"`);
+  const listing = await client.listFiles();
+  if (!listing.find(s3obj => s3obj.key === filename)) {
+    throw new Error(`File "${filename}" has not been uploaded`);
+  }
+
+  log(`Check contents of "${filename}"`);
+  const uploadedContents = await client.fetchFile(filename);
+  expectStringContains('"name": "handlebars"', uploadedContents);
+  await expectContentType(filename, 'application/octet-stream');
+
+  log(`Uploading with content type "${filename}"`);
+  await client.uploadFile('package.json', filename, {
+    contentType: 'text/html'
+  });
+  log('Checking content-type');
+  await expectContentType(filename, 'text/html');
+
+  log('Upload data as text/plain');
+  await client.uploadData('Hello world', filename, {
+    contentType: 'text/plain'
+  });
+  log('Checking content-type');
+  await expectContentType(filename, 'text/plain');
+  log(`Check contents of "${filename}"`);
+  expectStringContains('Hello world', await client.fetchFile(filename));
+  const helloWorldObj = (await client.listFiles()).find(
+    s3obj => s3obj.key === filename
+  );
+  assert.equal(helloWorldObj.size, 11, 'Checking file size of hello world');
+  assert.match(
+    helloWorldObj.lastModified,
+    ISO_DATE,
+    'Last modified must be an iso-date'
+  );
+
+  log(`Delete "${filename}"`);
+  await client.deleteFile(filename);
+
+  log(`Check if deleted "${filename}"`);
+  const foundFile = (await client.listFiles()).find(
+    s3obj => s3obj.key === filename
+  );
+  if (foundFile != null) {
+    throw new Error(`File "${filename}" has not been deleted`);
+  }
+});
+
+function expectStringContains(needle, haystack) {
+  if (!haystack.includes(needle)) {
+    throw new Error(`Expecting to find "${needle}" in string "${haystack}"`);
+  }
+}
+
+async function expectContentType(remoteName, expectedContentType) {
+  const contentType = (await fetch(client.fileUrl(remoteName))).headers.get(
+    'Content-Type'
+  );
+  if (contentType !== expectedContentType) {
+    throw new Error(
+      `Expecting to find content-type "${expectedContentType}" but found "${contentType}"`
+    );
+  }
+}
@@ -0,0 +1,31 @@
+const { PutObjectCommand } = require('@aws-sdk/client-s3');
+const fs = require('node:fs/promises');
+
+async function uploadFile(
+  s3Client,
+  bucket,
+  localName,
+  remoteName,
+  { contentType } = {}
+) {
+  const fileContents = await fs.readFile(localName);
+  await uploadData(s3Client, bucket, fileContents, remoteName, { contentType });
+}
+
+async function uploadData(
+  s3Client,
+  bucket,
+  data,
+  remoteName,
+  { contentType } = {}
+) {
+  const command = new PutObjectCommand({
+    Bucket: bucket,
+    Key: remoteName,
+    Body: data,
+    ContentType: contentType
+  });
+  await s3Client.send(command);
+}
+
+module.exports = { uploadFile, uploadData };
@@ -0,0 +1,37 @@
+/* eslint-disable no-console */
+const { createS3Client } = require('../s3client/index');
+
+const s3Client = createS3Client();
+
+function runTest(asyncFn) {
+  asyncFn({ log: console.log.bind(console) })
+    .finally(detectSurplusFiles)
+    .then(() => {
+      console.log('DONE');
+    })
+    .catch(error => {
+      console.error(error);
+      process.exit(1);
+    });
+}
+
+async function detectSurplusFiles() {
+  const listing = await s3Client.listFiles();
+  let surplusFileDetected = false;
+  const testFilesInBucket = listing.filter(name =>
+    name.key.includes('test-file')
+  );
+  for (const { key: filename } of testFilesInBucket) {
+    if (process.argv[2] === '--delete-surplus') {
+      await s3Client.deleteFile(filename);
+    } else {
+      console.log(`Detected surplus file "${filename}"`);
+      surplusFileDetected = true;
+    }
+  }
+  if (surplusFileDetected) {
+    console.log(`run with --delete-surplus to delete surplus files`);
+  }
+}
+
+module.exports = { runTest };
@@ -1,7 +1,8 @@
-const AWS = require('aws-sdk');
 const git = require('./util/git');
 const { createRegisterAsyncTaskFn } = require('./util/async-grunt-task');
 const semver = require('semver');
+const { publishWithSuffixes } = require('./aws-s3-builds-page/publish');
+const { generateFileList } = require('./aws-s3-builds-page/generateFileList');

 module.exports = function(grunt) {
  const registerAsyncTask = createRegisterAsyncTaskFn(grunt);
@@ -27,76 +28,8 @@ module.exports = function(grunt) {
    }

    if (suffixes.length > 0) {
-      initSDK();
-      grunt.log.writeln(
-        'publishing file-suffixes: ' + JSON.stringify(suffixes)
-      );
-      await publish(suffixes);
+      await publishWithSuffixes(suffixes);
+      await generateFileList('index');
    }
  });
-
-  function initSDK() {
-    const bucket = process.env.S3_BUCKET_NAME,
-      key = process.env.S3_ACCESS_KEY_ID,
-      secret = process.env.S3_SECRET_ACCESS_KEY;
-
-    if (!bucket || !key || !secret) {
-      throw new Error('Missing S3 config values');
-    }
-
-    AWS.config.update({ accessKeyId: key, secretAccessKey: secret });
-  }
-
-  async function publish(suffixes) {
-    const publishPromises = suffixes.map(suffix => publishSuffix(suffix));
-    return Promise.all(publishPromises);
-  }
-
-  async function publishSuffix(suffix) {
-    const filenames = [
-      'handlebars.js',
-      'handlebars.min.js',
-      'handlebars.runtime.js',
-      'handlebars.runtime.min.js'
-    ];
-    const publishPromises = filenames.map(async filename => {
-      const nameInBucket = getNameInBucket(filename, suffix);
-      const localFile = getLocalFile(filename);
-      await uploadToBucket(localFile, nameInBucket);
-      grunt.log.writeln(
-        `Published ${localFile} to build server (${nameInBucket})`
-      );
-    });
-    return Promise.all(publishPromises);
-  }
-
-  async function uploadToBucket(localFile, nameInBucket) {
-    const bucket = process.env.S3_BUCKET_NAME;
-    const uploadParams = {
-      Bucket: bucket,
-      Key: nameInBucket,
-      Body: grunt.file.read(localFile)
-    };
-    return s3PutObject(uploadParams);
-  }
 };
-
-function s3PutObject(uploadParams) {
-  const s3 = new AWS.S3();
-  return new Promise((resolve, reject) => {
-    s3.putObject(uploadParams, err => {
-      if (err != null) {
-        return reject(err);
-      }
-      resolve();
-    });
-  });
-}
-
-function getNameInBucket(filename, suffix) {
-  return filename.replace(/\.js$/, suffix + '.js');
-}
-
-function getLocalFile(filename) {
-  return 'dist/' + filename;
-}
Author	SHA1	Message	Date
Nils Knappmeier	f637dc76bd	remove old aws-sdk	2023-08-06 01:20:58 +02:00
Nils Knappmeier	e8299e0485	generate file list after publish	2023-08-06 01:17:42 +02:00
Nils Knappmeier	7989a5e9f9	function to upload a file list	2023-08-05 18:26:51 +02:00
Nils Knappmeier	bde5506b10	more functions that help uploading a file listing.	2023-08-05 18:09:54 +02:00
Nils Knappmeier	6118a3c829	fixup: revert accidental webpack-config changes	2023-08-05 15:42:32 +02:00
Nils Knappmeier	c5e5cad579	implement publish workflow	2023-08-05 15:11:00 +02:00
Nils Knappmeier	1c75903eaf	add delete and upload functions	2023-08-05 13:57:00 +02:00
Nils Knappmeier	ab920cc1ce	wip: refactor s3 access and generate file listing on build	2023-08-05 00:11:49 +02:00