From f2d8077d8b5a7060ee2a6a7bf1d7e58a754641b3 Mon Sep 17 00:00:00 2001 From: Dennis Reimann Date: Sat, 1 Mar 2025 10:29:31 +0100 Subject: [PATCH] Refactor feed generating and file revisioning --- helpers.js | 10 +++- package-lock.json | 102 +++++++++++++++++++++++++++++++++++++++++ package.json | 9 ++-- tasks/fetch_feed.js | 40 ++-------------- tasks/generate_feed.js | 29 ++++++++++++ tasks/replace_revs.js | 16 +++++++ 6 files changed, 167 insertions(+), 39 deletions(-) create mode 100644 tasks/generate_feed.js create mode 100644 tasks/replace_revs.js diff --git a/helpers.js b/helpers.js index 265e3318d32..2a3acdb7156 100644 --- a/helpers.js +++ b/helpers.js @@ -1,5 +1,8 @@ const { decode, encode } = require('html-entities') +const { writeFileSync } = require('fs') +const { join, resolve } = require('path') const meta = require('./content/meta.json') +const dir = resolve(__dirname, '.') // configure markdown-it const transformer = require('jstransformer') @@ -86,6 +89,9 @@ const assetUrl = (path, protocol = 'https') => { return url } +const write = (name, data) => writeFileSync(join(dir, name), data) +const writeJSON = (name, data) => write(`generated/${name}.json`, JSON.stringify(data, null, 2)) + module.exports = { markdown: mdTransformer.render, random, @@ -100,5 +106,7 @@ module.exports = { truncate, participantsWithAliases, participantToId, - toMeetupMapInfo + toMeetupMapInfo, + write, + writeJSON } diff --git a/package-lock.json b/package-lock.json index 5b132db9efc..3c948bad0b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,6 +33,7 @@ "postcss-mixins": "11.0.3", "postcss-nesting": "13.0.1", "pug": "3.0.3", + "replace-in-file": "6.3.5", "sync-request": "6.1.0", "xml-formatter": "3.6.4" } @@ -2189,6 +2190,12 @@ "universalify": "^0.1.0" } }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -2630,6 +2637,17 @@ "node": ">=8" } }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -3845,6 +3863,15 @@ "node": ">= 0.8" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, "node_modules/onchange": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/onchange/-/onchange-7.1.0.tgz", @@ -3954,6 +3981,15 @@ "node": ">=8" } }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -4822,6 +4858,66 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/replace-in-file": { + "version": "6.3.5", + "resolved": "https://registry.npmjs.org/replace-in-file/-/replace-in-file-6.3.5.tgz", + "integrity": "sha512-arB9d3ENdKva2fxRnSjwBEXfK1npgyci7ZZuwysgAp7ORjHSyxz6oqIjTEv8R0Ydl4Ll7uOAZXL4vbkhGIizCg==", + "dev": true, + "dependencies": { + "chalk": "^4.1.2", + "glob": "^7.2.0", + "yargs": "^17.2.1" + }, + "bin": { + "replace-in-file": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/replace-in-file/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/replace-in-file/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/replace-in-file/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -6372,6 +6468,12 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, "node_modules/ws": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", diff --git a/package.json b/package.json index f60eac9fae1..3c8b78f3962 100644 --- a/package.json +++ b/package.json @@ -18,19 +18,21 @@ "init": "run-s clean copy && run-p fetch build:data", "start": "cross-env NODE_ENV=development npm run init && cross-env NODE_ENV=development run-p start:*", "start:data": "onchange -k 'content/**/*' 'tasks/generate_site_data.js' -- npm run build:data", + "start:feed": "onchange -i -k 'generated/feed.json' 'tasks/generate_feed.js' -- npm run build:feed", "start:pages": "onchange -i -k 'pug.config.js' 'markdown.js' 'content/**' 'generated/**' 'src/**/*.pug' 'tasks/generate_pages.js' -- npm run build:pages -- {{file}}", "start:styles": "onchange -i -k 'src/**/*.css' -- npm run build:styles", "start:serve": "browser-sync start --config browser-sync.config.js --watch", "build": "npm run init && run-p build:*", "build:data": "node tasks/generate_site_data.js", + "build:feed": "node tasks/generate_feed.js", "build:nostr": "node tasks/generate_nostr.js", "build:pages": "node tasks/generate_pages.js", "build:styles": "postcss src/css/main.css --output dist/css/main.css", - "optimize": "run-p optimize:* && run-s rev", + "optimize": "run-s optimize:styles optimize:rev", "optimize:styles": "csso dist/css/main.css --output dist/css/main.css", - "rev": "node-file-rev --manifest=generated/rev.json --root=dist dist/css/* dist/js/* dist/img/*.svg dist/img/cover/*.png dist/img/ln/*.svg dist/img/participants/*.jpg dist/img/participants/*.png", + "optimize:rev": "node-file-rev --manifest=generated/rev.json --root=dist dist/css/* dist/js/* dist/img/*.svg dist/img/cover/*.png dist/img/ln/*.svg dist/img/participants/*.jpg dist/img/participants/*.png && node tasks/replace_revs.js", "sitemap": "node tasks/generate_sitemap.js", - "prod": "cross-env NODE_ENV=production run-s build optimize build:pages sitemap" + "prod": "cross-env NODE_ENV=production run-s build optimize sitemap" }, "dependencies": { "amplitudejs": "5.3.2" @@ -57,6 +59,7 @@ "postcss-mixins": "11.0.3", "postcss-nesting": "13.0.1", "pug": "3.0.3", + "replace-in-file": "6.3.5", "sync-request": "6.1.0", "xml-formatter": "3.6.4" } diff --git a/tasks/fetch_feed.js b/tasks/fetch_feed.js index 830e79e39ab..1133852c4c3 100644 --- a/tasks/fetch_feed.js +++ b/tasks/fetch_feed.js @@ -1,29 +1,18 @@ -const { writeFileSync } = require('fs') -const { join, resolve } = require('path') -const { replacements, slugify, stripHTML, participantsWithAliases, participantToId, assetUrl } = require('../helpers') +const { replacements, slugify, stripHTML, participantsWithAliases, participantToId, assetUrl, writeJSON } = require('../helpers') const { masterFeedUrl, publicFeedUrl, nodeId } = require('../content/meta.json') const participantsRaw = require('../content/participants.json') const request = require('sync-request') -const { XMLParser, XMLBuilder, XMLValidator } = require('fast-xml-parser') -const xmlFormat = require('xml-formatter') +const { XMLParser } = require('fast-xml-parser') const debug = process.env.CI -const dir = resolve(__dirname, '..') -const write = (name, data) => writeFileSync(join(dir, name), data) -const writeJSON = (name, data) => - write(`generated/${name}.json`, JSON.stringify(data, null, 2)) const participants = participantsWithAliases(participantsRaw) -const commonOpts = { +const xml2jsonOpts = { attributeNamePrefix: '', attributesGroupName: '__attr', ignoreAttributes: false, - cdataPropName: '__cdata' -} - -const xml2jsonOpts = { - ...commonOpts, + cdataPropName: '__cdata', parseTagValue: true, parseAttributeValue: false, trimValues: true, @@ -31,15 +20,9 @@ const xml2jsonOpts = { arrayMode: false } -const json2xmlOpts = { - ...commonOpts, - indentBy: ' ' -} - const regexBlockzeit = /Blockzeit\s(\d+(?:\.)?\d+)/ const parser = new XMLParser(xml2jsonOpts, true) -const builder = new XMLBuilder(json2xmlOpts) const parseEpisode = e => { const guid = e.guid['#text'] @@ -223,23 +206,10 @@ const parseEpisode = e => { return updated }) - const outputXML = builder.build(feed) + writeJSON('feed', feed) writeJSON('episodes', episodes) console.log('Neueste Episode:', episodes[0].title, '-', episodes[0].date) - const validation = XMLValidator.validate(outputXML) - if (validation) { - write( - 'dist/feed.xml', - xmlFormat(outputXML, { - indentation: json2xmlOpts.indentBy, - collapseContent: true - }) - ) - } else { - console.error(validation.err) - } - if (_noParticipants.length) { console.log('Keine Teilnehmerliste') console.table(_noParticipants) diff --git a/tasks/generate_feed.js b/tasks/generate_feed.js new file mode 100644 index 00000000000..798a5e66fce --- /dev/null +++ b/tasks/generate_feed.js @@ -0,0 +1,29 @@ + +const { XMLBuilder, XMLValidator } = require('fast-xml-parser') +const xmlFormat = require('xml-formatter') +const { write } = require('../helpers') +const feed = require('../generated/feed.json') + +const json2xmlOpts = { + attributeNamePrefix: '', + attributesGroupName: '__attr', + ignoreAttributes: false, + cdataPropName: '__cdata', + indentBy: ' ' +} +const builder = new XMLBuilder(json2xmlOpts) + +// Load and adapt feed +const outputXML = builder.build(feed) +const validation = XMLValidator.validate(outputXML) +if (validation) { + write( + 'dist/feed.xml', + xmlFormat(outputXML, { + indentation: json2xmlOpts.indentBy, + collapseContent: true + }) + ) +} else { + console.error(validation.err) +} diff --git a/tasks/replace_revs.js b/tasks/replace_revs.js new file mode 100644 index 00000000000..d88a8a9f5dd --- /dev/null +++ b/tasks/replace_revs.js @@ -0,0 +1,16 @@ +const { resolve } = require('path') +const { replaceInFileSync } = require('replace-in-file') +const rev = require('../generated/rev.json') +const options = { + files: [resolve(__dirname, '../dist/**/*.xml'), resolve(__dirname, '../dist/**/*.html')], + from: Object.keys(rev).map(key => new RegExp(key, 'g')), + to: Object.values(rev) +} + +try { + const results = replaceInFileSync(options) + console.log('Replacement results:', results.filter(result => result.hasChanged).length, 'files changed') +} +catch (error) { + console.error('Replacement error occurred:', error) +}