Refactor feed generating and file revisioning

helpers.js

@@ -1,5 +1,8 @@
 const { decode, encode } = require('html-entities')
+const { writeFileSync } = require('fs')
+const { join, resolve } = require('path')
 const meta = require('./content/meta.json')
+const dir = resolve(__dirname, '.')
 
 // configure markdown-it
 const transformer = require('jstransformer')
@@ -86,6 +89,9 @@ const assetUrl = (path, protocol = 'https') => {
   return url
 }
 
+const write = (name, data) => writeFileSync(join(dir, name), data)
+const writeJSON = (name, data) => write(`generated/${name}.json`, JSON.stringify(data, null, 2))
+
 module.exports = {
   markdown: mdTransformer.render,
   random,
@@ -100,5 +106,7 @@ module.exports = {
   truncate,
   participantsWithAliases,
   participantToId,
-  toMeetupMapInfo
+  toMeetupMapInfo,
+  write,
+  writeJSON
 }

package-lock.json

@@ -33,6 +33,7 @@
         "postcss-mixins": "11.0.3",
         "postcss-nesting": "13.0.1",
         "pug": "3.0.3",
+        "replace-in-file": "6.3.5",
         "sync-request": "6.1.0",
         "xml-formatter": "3.6.4"
       }
@@ -2189,6 +2190,12 @@
         "universalify": "^0.1.0"
       }
     },
+    "node_modules/fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
+      "dev": true
+    },
     "node_modules/fsevents": {
       "version": "2.3.3",
       "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
@@ -2630,6 +2637,17 @@
         "node": ">=8"
       }
     },
+    "node_modules/inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+      "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
+      "dev": true,
+      "dependencies": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
     "node_modules/inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
@@ -3845,6 +3863,15 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "dev": true,
+      "dependencies": {
+        "wrappy": "1"
+      }
+    },
     "node_modules/onchange": {
       "version": "7.1.0",
       "resolved": "https://registry.npmjs.org/onchange/-/onchange-7.1.0.tgz",
@@ -3954,6 +3981,15 @@
         "node": ">=8"
       }
     },
+    "node_modules/path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/path-key": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
@@ -4822,6 +4858,66 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/replace-in-file": {
+      "version": "6.3.5",
+      "resolved": "https://registry.npmjs.org/replace-in-file/-/replace-in-file-6.3.5.tgz",
+      "integrity": "sha512-arB9d3ENdKva2fxRnSjwBEXfK1npgyci7ZZuwysgAp7ORjHSyxz6oqIjTEv8R0Ydl4Ll7uOAZXL4vbkhGIizCg==",
+      "dev": true,
+      "dependencies": {
+        "chalk": "^4.1.2",
+        "glob": "^7.2.0",
+        "yargs": "^17.2.1"
+      },
+      "bin": {
+        "replace-in-file": "bin/cli.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/replace-in-file/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/replace-in-file/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "deprecated": "Glob versions prior to v9 are no longer supported",
+      "dev": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/replace-in-file/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -6372,6 +6468,12 @@
         "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
+    "node_modules/wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+      "dev": true
+    },
     "node_modules/ws": {
       "version": "8.17.1",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",

package.json

@@ -18,19 +18,21 @@
     "init": "run-s clean copy && run-p fetch build:data",
     "start": "cross-env NODE_ENV=development npm run init && cross-env NODE_ENV=development run-p start:*",
     "start:data": "onchange -k 'content/**/*' 'tasks/generate_site_data.js' -- npm run build:data",
+    "start:feed": "onchange -i -k 'generated/feed.json' 'tasks/generate_feed.js' -- npm run build:feed",
     "start:pages": "onchange -i -k 'pug.config.js' 'markdown.js' 'content/**' 'generated/**' 'src/**/*.pug' 'tasks/generate_pages.js' -- npm run build:pages -- {{file}}",
     "start:styles": "onchange -i -k 'src/**/*.css' -- npm run build:styles",
     "start:serve": "browser-sync start --config browser-sync.config.js --watch",
     "build": "npm run init && run-p build:*",
     "build:data": "node tasks/generate_site_data.js",
+    "build:feed": "node tasks/generate_feed.js",
     "build:nostr": "node tasks/generate_nostr.js",
     "build:pages": "node tasks/generate_pages.js",
     "build:styles": "postcss src/css/main.css --output dist/css/main.css",
-    "optimize": "run-p optimize:* && run-s rev",
+    "optimize": "run-s optimize:styles optimize:rev",
     "optimize:styles": "csso dist/css/main.css --output dist/css/main.css",
-    "rev": "node-file-rev --manifest=generated/rev.json --root=dist dist/css/* dist/js/* dist/img/*.svg dist/img/cover/*.png dist/img/ln/*.svg dist/img/participants/*.jpg dist/img/participants/*.png",
+    "optimize:rev": "node-file-rev --manifest=generated/rev.json --root=dist dist/css/* dist/js/* dist/img/*.svg dist/img/cover/*.png dist/img/ln/*.svg dist/img/participants/*.jpg dist/img/participants/*.png && node tasks/replace_revs.js",
     "sitemap": "node tasks/generate_sitemap.js",
-    "prod": "cross-env NODE_ENV=production run-s build optimize build:pages sitemap"
+    "prod": "cross-env NODE_ENV=production run-s build optimize sitemap"
   },
   "dependencies": {
     "amplitudejs": "5.3.2"
@@ -57,6 +59,7 @@
     "postcss-mixins": "11.0.3",
     "postcss-nesting": "13.0.1",
     "pug": "3.0.3",
+    "replace-in-file": "6.3.5",
     "sync-request": "6.1.0",
     "xml-formatter": "3.6.4"
   }

tasks/fetch_feed.js

@@ -1,29 +1,18 @@
-const { writeFileSync } = require('fs')
+const { replacements, slugify, stripHTML, participantsWithAliases, participantToId, assetUrl, writeJSON } = require('../helpers')
-const { join, resolve } = require('path')
-const { replacements, slugify, stripHTML, participantsWithAliases, participantToId, assetUrl } = require('../helpers')
 const { masterFeedUrl, publicFeedUrl, nodeId } = require('../content/meta.json')
 const participantsRaw = require('../content/participants.json')
 const request = require('sync-request')
-const { XMLParser, XMLBuilder, XMLValidator } = require('fast-xml-parser')
+const { XMLParser } = require('fast-xml-parser')
-const xmlFormat = require('xml-formatter')
 
 const debug = process.env.CI
-const dir = resolve(__dirname, '..')
-const write = (name, data) => writeFileSync(join(dir, name), data)
-const writeJSON = (name, data) =>
-  write(`generated/${name}.json`, JSON.stringify(data, null, 2))
 
 const participants = participantsWithAliases(participantsRaw)
 
-const commonOpts = {
+const xml2jsonOpts = {
   attributeNamePrefix: '',
   attributesGroupName: '__attr',
   ignoreAttributes: false,
-  cdataPropName: '__cdata'
+  cdataPropName: '__cdata',
-}
-
-const xml2jsonOpts = {
-  ...commonOpts,
   parseTagValue: true,
   parseAttributeValue: false,
   trimValues: true,
@@ -31,15 +20,9 @@ const xml2jsonOpts = {
   arrayMode: false
 }
 
-const json2xmlOpts = {
-  ...commonOpts,
-  indentBy: '  '
-}
-
 const regexBlockzeit = /Blockzeit\s(\d+(?:\.)?\d+)/
 
 const parser = new XMLParser(xml2jsonOpts, true)
-const builder = new XMLBuilder(json2xmlOpts)
 
 const parseEpisode = e => {
   const guid = e.guid['#text']
@@ -223,23 +206,10 @@ const parseEpisode = e => {
     return updated
   })
 
-  const outputXML = builder.build(feed)
+  writeJSON('feed', feed)
   writeJSON('episodes', episodes)
   console.log('Neueste Episode:', episodes[0].title, '-', episodes[0].date)
 
-  const validation = XMLValidator.validate(outputXML)
-  if (validation) {
-    write(
-      'dist/feed.xml',
-      xmlFormat(outputXML, {
-        indentation: json2xmlOpts.indentBy,
-        collapseContent: true
-      })
-    )
-  } else {
-    console.error(validation.err)
-  }
-
   if (_noParticipants.length) {
     console.log('Keine Teilnehmerliste')
     console.table(_noParticipants)

tasks/generate_feed.js

@@ -0,0 +1,29 @@
+
+const { XMLBuilder, XMLValidator } = require('fast-xml-parser')
+const xmlFormat = require('xml-formatter')
+const { write } = require('../helpers')
+const feed = require('../generated/feed.json')
+
+const json2xmlOpts = {
+  attributeNamePrefix: '',
+  attributesGroupName: '__attr',
+  ignoreAttributes: false,
+  cdataPropName: '__cdata',
+  indentBy: '  '
+}
+const builder = new XMLBuilder(json2xmlOpts)
+
+// Load and adapt feed
+const outputXML = builder.build(feed)
+const validation = XMLValidator.validate(outputXML)
+if (validation) {
+  write(
+    'dist/feed.xml',
+    xmlFormat(outputXML, {
+      indentation: json2xmlOpts.indentBy,
+      collapseContent: true
+    })
+  )
+} else {
+  console.error(validation.err)
+}

tasks/replace_revs.js

@@ -0,0 +1,16 @@
+const { resolve } = require('path')
+const { replaceInFileSync } = require('replace-in-file')
+const rev = require('../generated/rev.json')
+const options = {
+  files: [resolve(__dirname, '../dist/**/*.xml'), resolve(__dirname, '../dist/**/*.html')],
+  from: Object.keys(rev).map(key => new RegExp(key, 'g')),
+  to: Object.values(rev)
+}
+
+try {
+  const results = replaceInFileSync(options)
+  console.log('Replacement results:', results.filter(result => result.hasChanged).length, 'files changed')
+}
+catch (error) {
+  console.error('Replacement error occurred:', error)
+}