Refactor feed generating and file revisioning

2025-12-11 15:26:50 +00:00 · 2025-03-01 10:29:31 +01:00
parent 9028245a7c
commit f2d8077d8b
6 changed files with 167 additions and 39 deletions
--- a/helpers.js
+++ b/helpers.js
@@ -1,5 +1,8 @@
 const { decode, encode } = require('html-entities')
+const { writeFileSync } = require('fs')
+const { join, resolve } = require('path')
 const meta = require('./content/meta.json')
+const dir = resolve(__dirname, '.')

 // configure markdown-it
 const transformer = require('jstransformer')
@@ -86,6 +89,9 @@ const assetUrl = (path, protocol = 'https') => {
  return url
 }

+const write = (name, data) => writeFileSync(join(dir, name), data)
+const writeJSON = (name, data) => write(`generated/${name}.json`, JSON.stringify(data, null, 2))
+
 module.exports = {
  markdown: mdTransformer.render,
  random,
@@ -100,5 +106,7 @@ module.exports = {
  truncate,
  participantsWithAliases,
  participantToId,
-  toMeetupMapInfo
+  toMeetupMapInfo,
+  write,
+  writeJSON
 }
--- a/package-lock.json
+++ b/package-lock.json
@@ -33,6 +33,7 @@
        "postcss-mixins": "11.0.3",
        "postcss-nesting": "13.0.1",
        "pug": "3.0.3",
+        "replace-in-file": "6.3.5",
        "sync-request": "6.1.0",
        "xml-formatter": "3.6.4"
      }
@@ -2189,6 +2190,12 @@
        "universalify": "^0.1.0"
      }
    },
+    "node_modules/fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
+      "dev": true
+    },
    "node_modules/fsevents": {
      "version": "2.3.3",
      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
@@ -2630,6 +2637,17 @@
        "node": ">=8"
      }
    },
+    "node_modules/inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+      "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
+      "dev": true,
+      "dependencies": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
    "node_modules/inherits": {
      "version": "2.0.4",
      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
@@ -3845,6 +3863,15 @@
        "node": ">= 0.8"
      }
    },
+    "node_modules/once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "dev": true,
+      "dependencies": {
+        "wrappy": "1"
+      }
+    },
    "node_modules/onchange": {
      "version": "7.1.0",
      "resolved": "https://registry.npmjs.org/onchange/-/onchange-7.1.0.tgz",
@@ -3954,6 +3981,15 @@
        "node": ">=8"
      }
    },
+    "node_modules/path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/path-key": {
      "version": "3.1.1",
      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
@@ -4822,6 +4858,66 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/replace-in-file": {
+      "version": "6.3.5",
+      "resolved": "https://registry.npmjs.org/replace-in-file/-/replace-in-file-6.3.5.tgz",
+      "integrity": "sha512-arB9d3ENdKva2fxRnSjwBEXfK1npgyci7ZZuwysgAp7ORjHSyxz6oqIjTEv8R0Ydl4Ll7uOAZXL4vbkhGIizCg==",
+      "dev": true,
+      "dependencies": {
+        "chalk": "^4.1.2",
+        "glob": "^7.2.0",
+        "yargs": "^17.2.1"
+      },
+      "bin": {
+        "replace-in-file": "bin/cli.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/replace-in-file/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/replace-in-file/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "deprecated": "Glob versions prior to v9 are no longer supported",
+      "dev": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/replace-in-file/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
    "node_modules/require-directory": {
      "version": "2.1.1",
      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -6372,6 +6468,12 @@
        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
      }
    },
+    "node_modules/wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+      "dev": true
+    },
    "node_modules/ws": {
      "version": "8.17.1",
      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
--- a/package.json
+++ b/package.json
@@ -18,19 +18,21 @@
    "init": "run-s clean copy && run-p fetch build:data",
    "start": "cross-env NODE_ENV=development npm run init && cross-env NODE_ENV=development run-p start:*",
    "start:data": "onchange -k 'content/**/*' 'tasks/generate_site_data.js' -- npm run build:data",
+    "start:feed": "onchange -i -k 'generated/feed.json' 'tasks/generate_feed.js' -- npm run build:feed",
    "start:pages": "onchange -i -k 'pug.config.js' 'markdown.js' 'content/**' 'generated/**' 'src/**/*.pug' 'tasks/generate_pages.js' -- npm run build:pages -- {{file}}",
    "start:styles": "onchange -i -k 'src/**/*.css' -- npm run build:styles",
    "start:serve": "browser-sync start --config browser-sync.config.js --watch",
    "build": "npm run init && run-p build:*",
    "build:data": "node tasks/generate_site_data.js",
+    "build:feed": "node tasks/generate_feed.js",
    "build:nostr": "node tasks/generate_nostr.js",
    "build:pages": "node tasks/generate_pages.js",
    "build:styles": "postcss src/css/main.css --output dist/css/main.css",
-    "optimize": "run-p optimize:* && run-s rev",
+    "optimize": "run-s optimize:styles optimize:rev",
    "optimize:styles": "csso dist/css/main.css --output dist/css/main.css",
-    "rev": "node-file-rev --manifest=generated/rev.json --root=dist dist/css/* dist/js/* dist/img/*.svg dist/img/cover/*.png dist/img/ln/*.svg dist/img/participants/*.jpg dist/img/participants/*.png",
+    "optimize:rev": "node-file-rev --manifest=generated/rev.json --root=dist dist/css/* dist/js/* dist/img/*.svg dist/img/cover/*.png dist/img/ln/*.svg dist/img/participants/*.jpg dist/img/participants/*.png && node tasks/replace_revs.js",
    "sitemap": "node tasks/generate_sitemap.js",
-    "prod": "cross-env NODE_ENV=production run-s build optimize build:pages sitemap"
+    "prod": "cross-env NODE_ENV=production run-s build optimize sitemap"
  },
  "dependencies": {
    "amplitudejs": "5.3.2"
@@ -57,6 +59,7 @@
    "postcss-mixins": "11.0.3",
    "postcss-nesting": "13.0.1",
    "pug": "3.0.3",
+    "replace-in-file": "6.3.5",
    "sync-request": "6.1.0",
    "xml-formatter": "3.6.4"
  }
--- a/tasks/fetch_feed.js
+++ b/tasks/fetch_feed.js
@@ -1,29 +1,18 @@
-const { writeFileSync } = require('fs')
-const { join, resolve } = require('path')
-const { replacements, slugify, stripHTML, participantsWithAliases, participantToId, assetUrl } = require('../helpers')
+const { replacements, slugify, stripHTML, participantsWithAliases, participantToId, assetUrl, writeJSON } = require('../helpers')
 const { masterFeedUrl, publicFeedUrl, nodeId } = require('../content/meta.json')
 const participantsRaw = require('../content/participants.json')
 const request = require('sync-request')
-const { XMLParser, XMLBuilder, XMLValidator } = require('fast-xml-parser')
-const xmlFormat = require('xml-formatter')
+const { XMLParser } = require('fast-xml-parser')

 const debug = process.env.CI
-const dir = resolve(__dirname, '..')
-const write = (name, data) => writeFileSync(join(dir, name), data)
-const writeJSON = (name, data) =>
-  write(`generated/${name}.json`, JSON.stringify(data, null, 2))

 const participants = participantsWithAliases(participantsRaw)

-const commonOpts = {
+const xml2jsonOpts = {
  attributeNamePrefix: '',
  attributesGroupName: '__attr',
  ignoreAttributes: false,
-  cdataPropName: '__cdata'
-}
-
-const xml2jsonOpts = {
-  ...commonOpts,
+  cdataPropName: '__cdata',
  parseTagValue: true,
  parseAttributeValue: false,
  trimValues: true,
@@ -31,15 +20,9 @@ const xml2jsonOpts = {
  arrayMode: false
 }

-const json2xmlOpts = {
-  ...commonOpts,
-  indentBy: '  '
-}
-
 const regexBlockzeit = /Blockzeit\s(\d+(?:\.)?\d+)/

 const parser = new XMLParser(xml2jsonOpts, true)
-const builder = new XMLBuilder(json2xmlOpts)

 const parseEpisode = e => {
  const guid = e.guid['#text']
@@ -223,23 +206,10 @@ const parseEpisode = e => {
    return updated
  })

-  const outputXML = builder.build(feed)
+  writeJSON('feed', feed)
  writeJSON('episodes', episodes)
  console.log('Neueste Episode:', episodes[0].title, '-', episodes[0].date)

-  const validation = XMLValidator.validate(outputXML)
-  if (validation) {
-    write(
-      'dist/feed.xml',
-      xmlFormat(outputXML, {
-        indentation: json2xmlOpts.indentBy,
-        collapseContent: true
-      })
-    )
-  } else {
-    console.error(validation.err)
-  }
-
  if (_noParticipants.length) {
    console.log('Keine Teilnehmerliste')
    console.table(_noParticipants)
--- a/tasks/generate_feed.js
+++ b/tasks/generate_feed.js
@@ -0,0 +1,29 @@
+
+const { XMLBuilder, XMLValidator } = require('fast-xml-parser')
+const xmlFormat = require('xml-formatter')
+const { write } = require('../helpers')
+const feed = require('../generated/feed.json')
+
+const json2xmlOpts = {
+  attributeNamePrefix: '',
+  attributesGroupName: '__attr',
+  ignoreAttributes: false,
+  cdataPropName: '__cdata',
+  indentBy: '  '
+}
+const builder = new XMLBuilder(json2xmlOpts)
+
+// Load and adapt feed
+const outputXML = builder.build(feed)
+const validation = XMLValidator.validate(outputXML)
+if (validation) {
+  write(
+    'dist/feed.xml',
+    xmlFormat(outputXML, {
+      indentation: json2xmlOpts.indentBy,
+      collapseContent: true
+    })
+  )
+} else {
+  console.error(validation.err)
+}
--- a/tasks/replace_revs.js
+++ b/tasks/replace_revs.js
@@ -0,0 +1,16 @@
+const { resolve } = require('path')
+const { replaceInFileSync } = require('replace-in-file')
+const rev = require('../generated/rev.json')
+const options = {
+  files: [resolve(__dirname, '../dist/**/*.xml'), resolve(__dirname, '../dist/**/*.html')],
+  from: Object.keys(rev).map(key => new RegExp(key, 'g')),
+  to: Object.values(rev)
+}
+
+try {
+  const results = replaceInFileSync(options)
+  console.log('Replacement results:', results.filter(result => result.hasChanged).length, 'files changed')
+}
+catch (error) {
+  console.error('Replacement error occurred:', error)
+}