diff options
Diffstat (limited to 'src/upd8.js')
| -rwxr-xr-x | src/upd8.js | 4394 |
1 files changed, 1646 insertions, 2748 deletions
diff --git a/src/upd8.js b/src/upd8.js index e9b3d4e..6bd52da 100755 --- a/src/upd8.js +++ b/src/upd8.js @@ -17,6 +17,9 @@ // going to 8e in. May8e JSON, 8ut more likely some weird custom format // which will 8e a lot easier to edit. // +// Like three years later oh god: SURPISE! We went with the latter, but +// they're YAML now. Probably. Assuming that hasn't changed, yet. +// // 3. Generate the page files! They're just static index.html files, and are // what gh-pages (or wherever this is hosted) will show to clients. // Hopefully pretty minimalistic HTML, 8ut like, shrug. They'll reference @@ -28,3088 +31,1983 @@ // Oh yeah, like. Just run this through some relatively recent version of // node.js and you'll 8e fine. ...Within the project root. O8viously. -// HEY FUTURE ME!!!!!!!! Don't forget to implement artist pages! Those are, -// like, the coolest idea you've had yet, so DO NOT FORGET. (Remem8er, link -// from track listings, etc!) --- Thanks, past me. To futurerer me: an al8um -// listing page (a list of all the al8ums)! Make sure to sort these 8y date - -// we'll need a new field for al8ums. - -// ^^^^^^^^ DID THAT! 8ut also, artist images. Pro8a8ly stolen from the fandom -// wiki (I found half those images anywayz). - -// TRACK ART CREDITS. This is a must. - -// 2020-08-23 -// ATTENTION ALL 8*TCHES AND OTHER GENDER TRUCKERS: AS IT TURNS OUT, THIS CODE -// ****SUCKS****. I DON'T THINK ANYTHING WILL EVER REDEEM IT, 8UT THAT DOESN'T -// MEAN WE CAN'T TAKE SOME ACTION TO MAKE WRITING IT A LITTLE LESS TERRI8LE. -// We're gonna start defining STRUCTURES to make things suck less!!!!!!!! -// No classes 8ecause those are a huge pain and like, pro8a8ly 8ad performance -// or whatever -- just some standard structures that should 8e followed -// wherever reasona8le. Only one I need today is the contri8 one 8ut let's put -// any new general-purpose structures here too, ok? -// -// Contri8ution: {who, what, date, thing}. D8 and thing are the new fields. -// -// Use these wisely, which is to say all the time and instead of whatever -// terri8le new pseudo structure you're trying to invent!!!!!!!! -// -// Upd8 2021-01-03: Soooooooo we didn't actually really end up using these, -// lol? Well there's still only one anyway. Kinda ended up doing a 8ig refactor -// of all the o8ject structures today. It's not *especially* relevant 8ut feels -// worth mentioning? I'd get rid of this comment 8lock 8ut I like it too much! -// Even though I haven't actually reread it, lol. 8ut yeah, hopefully in the -// spirit of this "make things more consistent" attitude I 8rought up 8ack in -// August, stuff's lookin' 8etter than ever now. W00t! - -import * as path from 'path'; -import { promisify } from 'util'; -import { fileURLToPath } from 'url'; - -// I made this dependency myself! A long, long time ago. It is pro8a8ly my -// most useful li8rary ever. I'm not sure 8esides me actually uses it, though. -import fixWS from 'fix-whitespace'; -// Wait nevermind, I forgot a8out why-do-kids-love-the-taste-of-cinnamon-toast- -// crunch. THAT is my 8est li8rary. - -// It stands for "HTML Entities", apparently. Cursed. -import he from 'he'; +import {execSync} from 'node:child_process'; +import {readdir, readFile} from 'node:fs/promises'; +import * as path from 'node:path'; +import {fileURLToPath} from 'node:url'; -import { - // This is the dum8est name for a function possi8le. Like, SURE, fine, may8e - // the UNIX people had some valid reason to go with the weird truncated - // lowercased convention they did. 8ut Node didn't have to ALSO use that - // convention! Would it have 8een so hard to just name the function - // something like fs.readDirectory???????? No, it wouldn't have 8een. - readdir, - // ~~ 8ut okay, like, look at me. DOING THE SAME THING. See, *I* could have - // named my promisified function differently, and yet I did not. I literally - // cannot explain why. We are all used to following in the 8ad decisions of - // our ancestors, and never never never never never never never consider - // that hey, may8e we don't need to make the exact same decisions they did. - // Even when we're perfectly aware th8t's exactly what we're doing! ~~ - // - // 2021 ADDENDUM: Ok, a year and a half later the a8ove is still true, - // except for the part a8out promisifying, since fs/promises - // already does that for us. 8ut I could STILL import it - // using my own name (`readdir as readDirectory`), and yet - // here I am, defin8tely not doing that. - // SOME THINGS NEVER CHANGE. - // - // Programmers, including me, are all pretty stupid. - - // 8ut I mean, come on. Look. Node decided to use readFile, instead of like, - // what, cat? Why couldn't they rename readdir too???????? As Johannes - // Kepler once so elegantly put it: "Shrug." - readFile, - writeFile, - access, - mkdir, - symlink, - unlink -} from 'fs/promises'; - -import genThumbs from './gen-thumbs.js'; -import { listingSpec, listingTargetSpec } from './listing-spec.js'; -import urlSpec from './url-spec.js'; -import * as pageSpecs from './page/index.js'; - -import find from './util/find.js'; -import * as html from './util/html.js'; -import unbound_link, {getLinkThemeString} from './util/link.js'; +import wrap from 'word-wrap'; -import { - fancifyFlashURL, - fancifyURL, - generateChronologyLinks, - generateCoverLink, - generateInfoGalleryLinks, - generatePreviousNextLinks, - getAlbumGridHTML, - getAlbumStylesheet, - getArtistString, - getFlashGridHTML, - getGridHTML, - getRevealStringFromTags, - getRevealStringFromWarnings, - getThemeString, - iconifyURL -} from './misc-templates.js'; +// Due to import time shenanigans, these imports have to come in the specified +// order. This obviously needs fixing up. +/* precede #find */ import { - decorateTime, - logWarn, - logInfo, - logError, - parseOptions, - progressPromiseAll -} from './util/cli.js'; + filterReferenceErrors, + reportDuplicateDirectories, + reportContentTextErrors, +} from '#data-checks'; + +import {bindFind, getAllFindSpecs} from '#find'; + +// End of import time shenanigans (hopefully) + +import {showAggregate} from '#aggregate'; +import CacheableObject from '#cacheable-object'; +import {displayCompositeCacheAnalysis} from '#composite'; +import {processLanguageFile, watchLanguageFile, internalDefaultStringsFile} + from '#language'; +import {isMain, traverse} from '#node-utils'; +import {sortByName} from '#sort'; +import {empty, withEntries} from '#sugar'; +import {generateURLs, urlSpec} from '#urls'; +import {linkWikiDataArrays, loadAndProcessDataDocuments, sortWikiDataArrays} + from '#yaml'; import { - validateReplacerSpec, - transformInline -} from './util/replacer.js'; + colors, + decorateTime, + fileIssue, + logWarn, + logInfo, + logError, + parseOptions, + progressCallAll, +} from '#cli'; + +import genThumbs, { + CACHE_FILE as thumbsCacheFile, + defaultMagickThreads, + determineMediaCachePath, + isThumb, + migrateThumbsIntoDedicatedCacheDirectory, + verifyImagePaths, +} from '#thumbs'; + +import FileSizePreloader from './file-size-preloader.js'; +import {listingSpec, listingTargetSpec} from './listing-spec.js'; +import * as buildModes from './write/build-modes/index.js'; -import { - genStrings, - count, - list -} from './util/strings.js'; +const __dirname = path.dirname(fileURLToPath(import.meta.url)); -import { - chunkByConditions, - chunkByProperties, - getAlbumCover, - getAlbumListTag, - getAllTracks, - getArtistCommentary, - getArtistNumContributions, - getFlashCover, - getKebabCase, - getTotalDuration, - getTrackCover, - sortByArtDate, - sortByDate, - sortByName -} from './util/wiki-data.js'; +const CACHEBUST = 23; -import { - serializeContribs, - serializeCover, - serializeGroupsForAlbum, - serializeGroupsForTrack, - serializeImagePaths, - serializeLink -} from './util/serialize.js'; +let COMMIT; +try { + COMMIT = execSync('git log --format="%h %B" -n 1 HEAD', {cwd: __dirname}).toString().trim(); +} catch (error) { + COMMIT = '(failed to detect)'; +} -import { - bindOpts, - call, - filterEmptyLines, - mapInPlace, - queue, - splitArray, - unique, - withEntries -} from './util/sugar.js'; +const BUILD_TIME = new Date(); -import { - generateURLs, - thumb -} from './util/urls.js'; +const STATUS_NOT_STARTED = `not started`; +const STATUS_NOT_APPLICABLE = `not applicable`; +const STATUS_STARTED_NOT_DONE = `started but not yet done`; +const STATUS_DONE_CLEAN = `done without warnings`; +const STATUS_FATAL_ERROR = `fatal error`; +const STATUS_HAS_WARNINGS = `has warnings`; -// Pensive emoji! -import { - FANDOM_GROUP_DIRECTORY, - OFFICIAL_GROUP_DIRECTORY, - UNRELEASED_TRACKS_DIRECTORY -} from './util/magic-constants.js'; +const defaultStepStatus = {status: STATUS_NOT_STARTED, annotation: null}; -const __dirname = path.dirname(fileURLToPath(import.meta.url)); +// Defined globally for quick access outside the main() function's contents. +// This will be initialized and mutated over the course of main(). +let stepStatusSummary; +let showStepStatusSummary = false; -const CACHEBUST = 7; - -const WIKI_INFO_FILE = 'wiki-info.txt'; -const HOMEPAGE_INFO_FILE = 'homepage.txt'; -const ARTIST_DATA_FILE = 'artists.txt'; -const FLASH_DATA_FILE = 'flashes.txt'; -const NEWS_DATA_FILE = 'news.txt'; -const TAG_DATA_FILE = 'tags.txt'; -const GROUP_DATA_FILE = 'groups.txt'; -const STATIC_PAGE_DATA_FILE = 'static-pages.txt'; -const DEFAULT_STRINGS_FILE = 'strings-default.json'; - -// Code that's common 8etween the 8uild code (i.e. upd8.js) and gener8ted -// site code should 8e put here. Which, uh, ~~only really means this one -// file~~ is now a variety of useful utilities! -// -// Rather than hard code it, anything in this directory can 8e shared across -// 8oth ends of the code8ase. -// (This gets symlinked into the --data directory.) -const UTILITY_DIRECTORY = 'util'; - -// Code that's used only in the static site! CSS, cilent JS, etc. -// (This gets symlinked into the --data directory.) -const STATIC_DIRECTORY = 'static'; - -// Su8directory under provided --data directory for al8um files, which are -// read from and processed to compose the majority of album and track data. -const DATA_ALBUM_DIRECTORY = 'album'; - -// Shared varia8les! These are more efficient to access than a shared varia8le -// (or at least I h8pe so), and are easier to pass across functions than a -// 8unch of specific arguments. -// -// Upd8: Okay yeah these aren't actually any different. Still cleaner than -// passing around a data object containing all this, though. -let dataPath; -let mediaPath; -let langPath; -let outputPath; - -// Glo8al data o8ject shared 8etween 8uild functions and all that. This keeps -// everything encapsul8ted in one place, so it's easy to pass and share across -// modules! -let wikiData = {}; - -let queueSize; - -let languages; - -const urls = generateURLs(urlSpec); - -// Note there isn't a 'find track data files' function. I plan on including the -// data for all tracks within an al8um collected in the single metadata file -// for that al8um. Otherwise there'll just 8e way too many files, and I'd also -// have to worry a8out linking track files to al8um files (which would contain -// only the track listing, not track data itself), and dealing with errors of -// missing track files (or track files which are not linked to al8ums). All a -// 8unch of stuff that's a pain to deal with for no apparent 8enefit. -async function findFiles(dataPath, filter = f => true) { - return (await readdir(dataPath)) - .map(file => path.join(dataPath, file)) - .filter(file => filter(file)); -} +async function main() { + Error.stackTraceLimit = Infinity; -function* getSections(lines) { - // ::::) - const isSeparatorLine = line => /^-{8,}$/.test(line); - yield* splitArray(lines, isSeparatorLine); -} + stepStatusSummary = { + determineMediaCachePath: + {...defaultStepStatus, name: `determine media cache path`}, -function getBasicField(lines, name) { - const line = lines.find(line => line.startsWith(name + ':')); - return line && line.slice(name.length + 1).trim(); -} + migrateThumbnails: + {...defaultStepStatus, name: `migrate thumbnails`}, -function getDimensionsField(lines, name) { - const string = getBasicField(lines, name); - if (!string) return string; - const parts = string.split(/[x,* ]+/g); - if (parts.length !== 2) throw new Error(`Invalid dimensions: ${string} (expected width & height)`); - const nums = parts.map(part => Number(part.trim())); - if (nums.includes(NaN)) throw new Error(`Invalid dimensions: ${string} (couldn't parse as numbers)`); - return nums; -} + loadThumbnailCache: + {...defaultStepStatus, name: `load thumbnail cache file`}, -function getBooleanField(lines, name) { - // The ?? oper8tor (which is just, hilariously named, lol) can 8e used to - // specify a default! - const value = getBasicField(lines, name); - switch (value) { - case 'yes': - case 'true': - return true; - case 'no': - case 'false': - return false; - default: - return null; - } -} + generateThumbnails: + {...defaultStepStatus, name: `generate thumbnails`}, -function getListField(lines, name) { - let startIndex = lines.findIndex(line => line.startsWith(name + ':')); - // If callers want to default to an empty array, they should stick - // "|| []" after the call. - if (startIndex === -1) { - return null; - } - // We increment startIndex 8ecause we don't want to include the - // "heading" line (e.g. "URLs:") in the actual data. - startIndex++; - let endIndex = lines.findIndex((line, index) => index >= startIndex && !line.startsWith('- ')); - if (endIndex === -1) { - endIndex = lines.length; - } - if (endIndex === startIndex) { - // If there is no list that comes after the heading line, treat the - // heading line itself as the comma-separ8ted array value, using - // the 8asic field function to do that. (It's l8 and my 8rain is - // sleepy. Please excuse any unhelpful comments I may write, or may - // have already written, in this st8. Thanks!) - const value = getBasicField(lines, name); - return value && value.split(',').map(val => val.trim()); - } - const listLines = lines.slice(startIndex, endIndex); - return listLines.map(line => line.slice(2)); -}; + loadDataFiles: + {...defaultStepStatus, name: `load and process data files`}, -function getContributionField(section, name) { - let contributors = getListField(section, name); + linkWikiDataArrays: + {...defaultStepStatus, name: `link wiki data arrays`}, - if (!contributors) { - return null; - } + precacheCommonData: + {...defaultStepStatus, name: `precache common data`}, - if (contributors.length === 1 && contributors[0].startsWith('<i>')) { - const arr = []; - arr.textContent = contributors[0]; - return arr; - } + reportDuplicateDirectories: + {...defaultStepStatus, name: `report duplicate directories`}, - contributors = contributors.map(contrib => { - // 8asically, the format is "Who (What)", or just "Who". 8e sure to - // keep in mind that "what" doesn't necessarily have a value! - const match = contrib.match(/^(.*?)( \((.*)\))?$/); - if (!match) { - return contrib; - } - const who = match[1]; - const what = match[3] || null; - return {who, what}; - }); + filterReferenceErrors: + {...defaultStepStatus, name: `filter reference errors`}, - const badContributor = contributors.find(val => typeof val === 'string'); - if (badContributor) { - return {error: `An entry has an incorrectly formatted contributor, "${badContributor}".`}; - } + reportContentTextErrors: + {...defaultStepStatus, name: `report content text errors`}, - if (contributors.length === 1 && contributors[0].who === 'none') { - return null; - } + sortWikiDataArrays: + {...defaultStepStatus, name: `sort wiki data arrays`}, - return contributors; -}; + precacheAllData: + {...defaultStepStatus, name: `precache nearly all data`}, -function getMultilineField(lines, name) { - // All this code is 8asically the same as the getListText - just with a - // different line prefix (four spaces instead of a dash and a space). - let startIndex = lines.findIndex(line => line.startsWith(name + ':')); - if (startIndex === -1) { - return null; - } - startIndex++; - let endIndex = lines.findIndex((line, index) => index >= startIndex && line.length && !line.startsWith(' ')); - if (endIndex === -1) { - endIndex = lines.length; - } - // If there aren't any content lines, don't return anything! - if (endIndex === startIndex) { - return null; - } - // We also join the lines instead of returning an array. - const listLines = lines.slice(startIndex, endIndex); - return listLines.map(line => line.slice(4)).join('\n'); -}; - -const replacerSpec = { - 'album': { - find: 'album', - link: 'album' - }, - 'album-commentary': { - find: 'album', - link: 'albumCommentary' - }, - 'artist': { - find: 'artist', - link: 'artist' - }, - 'artist-gallery': { - find: 'artist', - link: 'artistGallery' - }, - 'commentary-index': { - find: null, - link: 'commentaryIndex' - }, - 'date': { - find: null, - value: ref => new Date(ref), - html: (date, {strings}) => `<time datetime="${date.toString()}">${strings.count.date(date)}</time>` - }, - 'flash': { - find: 'flash', - link: 'flash', - transformName(name, node, input) { - const nextCharacter = input[node.iEnd]; - const lastCharacter = name[name.length - 1]; - if ( - ![' ', '\n', '<'].includes(nextCharacter) && - lastCharacter === '.' - ) { - return name.slice(0, -1); - } else { - return name; - } - } - }, - 'group': { - find: 'group', - link: 'groupInfo' - }, - 'group-gallery': { - find: 'group', - link: 'groupGallery' - }, - 'listing-index': { - find: null, - link: 'listingIndex' - }, - 'listing': { - find: 'listing', - link: 'listing' - }, - 'media': { - find: null, - link: 'media' - }, - 'news-index': { - find: null, - link: 'newsIndex' - }, - 'news-entry': { - find: 'newsEntry', - link: 'newsEntry' - }, - 'root': { - find: null, - link: 'root' - }, - 'site': { - find: null, - link: 'site' - }, - 'static': { - find: 'staticPage', - link: 'staticPage' - }, - 'string': { - find: null, - value: ref => ref, - html: (ref, {strings, args}) => strings(ref, args) - }, - 'tag': { - find: 'tag', - link: 'tag' - }, - 'track': { - find: 'track', - link: 'track' - } -}; + // TODO: This should be split into load/watch steps. + loadInternalDefaultLanguage: + {...defaultStepStatus, name: `load internal default language`}, -if (!validateReplacerSpec(replacerSpec, unbound_link)) { - process.exit(); -} - -function parseAttributes(string, {to}) { - const attributes = Object.create(null); - const skipWhitespace = i => { - const ws = /\s/; - if (ws.test(string[i])) { - const match = string.slice(i).match(/[^\s]/); - if (match) { - return i + match.index; - } else { - return string.length; - } - } else { - return i; - } - }; + loadLanguageFiles: + {...defaultStepStatus, name: `statically load custom language files`}, - for (let i = 0; i < string.length;) { - i = skipWhitespace(i); - const aStart = i; - const aEnd = i + string.slice(i).match(/[\s=]|$/).index; - const attribute = string.slice(aStart, aEnd); - i = skipWhitespace(aEnd); - if (string[i] === '=') { - i = skipWhitespace(i + 1); - let end, endOffset; - if (string[i] === '"' || string[i] === "'") { - end = string[i]; - endOffset = 1; - i++; - } else { - end = '\\s'; - endOffset = 0; - } - const vStart = i; - const vEnd = i + string.slice(i).match(new RegExp(`${end}|$`)).index; - const value = string.slice(vStart, vEnd); - i = vEnd + endOffset; - if (attribute === 'src' && value.startsWith('media/')) { - attributes[attribute] = to('media.path', value.slice('media/'.length)); - } else { - attributes[attribute] = value; - } - } else { - attributes[attribute] = attribute; - } - } - return Object.fromEntries(Object.entries(attributes).map(([ key, val ]) => [ - key, - val === 'true' ? true : - val === 'false' ? false : - val === key ? true : - val - ])); -} + watchLanguageFiles: + {...defaultStepStatus, name: `watch custom language files`}, -function transformMultiline(text, { - parseAttributes, - transformInline -}) { - // Heck yes, HTML magics. + initializeDefaultLanguage: + {...defaultStepStatus, name: `initialize default language`}, - text = transformInline(text.trim()); + verifyImagePaths: + {...defaultStepStatus, name: `verify missing/misplaced image paths`}, - const outLines = []; + preloadFileSizes: + {...defaultStepStatus, name: `preload file sizes`}, - const indentString = ' '.repeat(4); + performBuild: + {...defaultStepStatus, name: `perform selected build mode`}, + }; - let levelIndents = []; - const openLevel = indent => { - // opening a sublist is a pain: to be semantically *and* visually - // correct, we have to append the <ul> at the end of the existing - // previous <li> - const previousLine = outLines[outLines.length - 1]; - if (previousLine?.endsWith('</li>')) { - // we will re-close the <li> later - outLines[outLines.length - 1] = previousLine.slice(0, -5) + ' <ul>'; - } else { - // if the previous line isn't a list item, this is the opening of - // the first list level, so no need for indent - outLines.push('<ul>'); - } - levelIndents.push(indent); - }; - const closeLevel = () => { - levelIndents.pop(); - if (levelIndents.length) { - // closing a sublist, so close the list item containing it too - outLines.push(indentString.repeat(levelIndents.length) + '</ul></li>'); - } else { - // closing the final list level! no need for indent here - outLines.push('</ul>'); - } - }; + const defaultQueueSize = 500; - // okay yes we should support nested formatting, more than one blockquote - // layer, etc, but hear me out here: making all that work would basically - // be the same as implementing an entire markdown converter, which im not - // interested in doing lol. sorry!!! - let inBlockquote = false; - - for (let line of text.split(/\r|\n|\r\n/)) { - const imageLine = line.startsWith('<img'); - line = line.replace(/<img (.*?)>/g, (match, attributes) => img({ - lazy: true, - link: true, - thumb: 'medium', - ...parseAttributes(attributes) - })); - - let indentThisLine = 0; - let lineContent = line; - let lineTag = 'p'; - - const listMatch = line.match(/^( *)- *(.*)$/); - if (listMatch) { - // is a list item! - if (!levelIndents.length) { - // first level is always indent = 0, regardless of actual line - // content (this is to avoid going to a lesser indent than the - // initial level) - openLevel(0); - } else { - // find level corresponding to indent - const indent = listMatch[1].length; - let i; - for (i = levelIndents.length - 1; i >= 0; i--) { - if (levelIndents[i] <= indent) break; - } - // note: i cannot equal -1 because the first indentation level - // is always 0, and the minimum indentation is also 0 - if (levelIndents[i] === indent) { - // same indent! return to that level - while (levelIndents.length - 1 > i) closeLevel(); - // (if this is already the current level, the above loop - // will do nothing) - } else if (levelIndents[i] < indent) { - // lesser indent! branch based on index - if (i === levelIndents.length - 1) { - // top level is lesser: add a new level - openLevel(indent); - } else { - // lower level is lesser: return to that level - while (levelIndents.length - 1 > i) closeLevel(); - } - } - } - // finally, set variables for appending content line - indentThisLine = levelIndents.length; - lineContent = listMatch[2]; - lineTag = 'li'; - } else { - // not a list item! close any existing list levels - while (levelIndents.length) closeLevel(); - - // like i said, no nested shenanigans - quotes only appear outside - // of lists. sorry! - const quoteMatch = line.match(/^> *(.*)$/); - if (quoteMatch) { - // is a quote! open a blockquote tag if it doesnt already exist - if (!inBlockquote) { - inBlockquote = true; - outLines.push('<blockquote>'); - } - indentThisLine = 1; - lineContent = quoteMatch[1]; - } else if (inBlockquote) { - // not a quote! close a blockquote tag if it exists - inBlockquote = false; - outLines.push('</blockquote>'); - } - } + const buildModeFlagOptions = ( + withEntries(buildModes, entries => + entries.map(([key, mode]) => [key, { + help: mode.description, + type: 'flag', + }]))); - if (lineTag === 'p') { - // certain inline element tags should still be postioned within a - // paragraph; other elements (e.g. headings) should be added as-is - const elementMatch = line.match(/^<(.*?)[ >]/); - if (elementMatch && !imageLine && !['a', 'abbr', 'b', 'bdo', 'br', 'cite', 'code', 'data', 'datalist', 'del', 'dfn', 'em', 'i', 'img', 'ins', 'kbd', 'mark', 'output', 'picture', 'q', 'ruby', 'samp', 'small', 'span', 'strong', 'sub', 'sup', 'svg', 'time', 'var', 'wbr'].includes(elementMatch[1])) { - lineTag = ''; - } - } + const selectedBuildModeFlags = Object.keys( + await parseOptions(process.argv.slice(2), { + [parseOptions.handleUnknown]: () => {}, + ...buildModeFlagOptions, + })); - let pushString = indentString.repeat(indentThisLine); - if (lineTag) { - pushString += `<${lineTag}>${lineContent}</${lineTag}>`; - } else { - pushString += lineContent; - } - outLines.push(pushString); - } + let selectedBuildModeFlag; + let usingDefaultBuildMode; - // after processing all lines... + if (empty(selectedBuildModeFlags)) { + selectedBuildModeFlag = 'static-build'; + usingDefaultBuildMode = true; + } else if (selectedBuildModeFlags.length > 1) { + logError`Building multiple modes (${selectedBuildModeFlags.join(', ')}) at once not supported.`; + logError`Please specify a maximum of one build mode.`; + return false; + } else { + selectedBuildModeFlag = selectedBuildModeFlags[0]; + usingDefaultBuildMode = false; + } - // if still in a list, close all levels - while (levelIndents.length) closeLevel(); + const selectedBuildMode = buildModes[selectedBuildModeFlag]; - // if still in a blockquote, close its tag - if (inBlockquote) { - inBlockquote = false; - outLines.push('</blockquote>'); - } + // This is about to get a whole lot more stuff put in it. + const wikiData = { + listingSpec, + listingTargetSpec, + }; - return outLines.join('\n'); -} + const buildOptions = selectedBuildMode.getCLIOptions(); -function transformLyrics(text, { - transformInline, - transformMultiline -}) { - // Different from transformMultiline 'cuz it joins multiple lines together - // with line 8reaks (<br>); transformMultiline treats each line as its own - // complete paragraph (or list, etc). - - // If it looks like old data, then like, oh god. - // Use the normal transformMultiline tool. - if (text.includes('<br')) { - return transformMultiline(text); - } + const commonOptions = { + 'help': { + help: `Display usage info and basic information for the \`hsmusic\` command`, + type: 'flag', + }, - text = transformInline(text.trim()); - - let buildLine = ''; - const addLine = () => outLines.push(`<p>${buildLine}</p>`); - const outLines = []; - for (const line of text.split('\n')) { - if (line.length) { - if (buildLine.length) { - buildLine += '<br>'; - } - buildLine += line; - } else if (buildLine.length) { - addLine(); - buildLine = ''; - } - } - if (buildLine.length) { - addLine(); - } - return outLines.join('\n'); -} + // Data files for the site, including flash, artist, and al8um data, + // and like a jillion other things too. Pretty much everything which + // makes an individual wiki what it is goes here! + 'data-path': { + help: `Specify path to data directory, including YAML files that cover all info about wiki content, layout, and structure\n\nAlways required for wiki building, but may be provided via the HSMUSIC_DATA environment variable instead`, + type: 'value', + }, -function getCommentaryField(lines) { - const text = getMultilineField(lines, 'Commentary'); - if (text) { - const lines = text.split('\n'); - if (!lines[0].replace(/<\/b>/g, '').includes(':</i>')) { - return {error: `An entry is missing commentary citation: "${lines[0].slice(0, 40)}..."`}; - } - return text; - } else { - return null; - } -}; + // Static media will 8e referenced in the site here! The contents are + // categorized; check out MEDIA_ALBUM_ART_DIRECTORY and other constants + // near the top of this file (upd8.js). + 'media-path': { + help: `Specify path to media directory, including album artwork and additional files, as well as custom site layout media and other media files for reference or linking in wiki content\n\nAlways required for wiki building, but may be provided via the HSMUSIC_MEDIA environment variable instead`, + type: 'value', + }, -async function processAlbumDataFile(file) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - // This function can return "error o8jects," which are really just - // ordinary o8jects with an error message attached. I'm not 8othering - // with error codes here or anywhere in this function; while this would - // normally 8e 8ad coding practice, it doesn't really matter here, - // 8ecause this isn't an API getting consumed 8y other services (e.g. - // translaction functions). If we return an error, the caller will just - // print the attached message in the output summary. - return {error: `Could not read ${file} (${error.code}).`}; - } + 'media-cache-path': { + help: `Specify path to media cache directory, including automatically generated thumbnails\n\nThis usually doesn't need to be provided, and will be inferred by adding "-cache" to the end of the media directory`, + type: 'value', + }, - // We're pro8a8ly supposed to, like, search for a header somewhere in the - // al8um contents, to make sure it's trying to 8e the intended structure - // and is a valid utf-8 (or at least ASCII) file. 8ut like, whatever. - // We'll just return more specific errors if it's missing necessary data - // fields. - - const contentLines = contents.split('\n'); - - // In this line of code I defeat the purpose of using a generator in the - // first place. Sorry!!!!!!!! - const sections = Array.from(getSections(contentLines)); - - const albumSection = sections[0]; - const album = {}; - - album.name = getBasicField(albumSection, 'Album'); - album.artists = getContributionField(albumSection, 'Artists') || getContributionField(albumSection, 'Artist'); - album.wallpaperArtists = getContributionField(albumSection, 'Wallpaper Art'); - album.wallpaperStyle = getMultilineField(albumSection, 'Wallpaper Style'); - album.bannerArtists = getContributionField(albumSection, 'Banner Art'); - album.bannerStyle = getMultilineField(albumSection, 'Banner Style'); - album.bannerDimensions = getDimensionsField(albumSection, 'Banner Dimensions'); - album.date = getBasicField(albumSection, 'Date'); - album.trackArtDate = getBasicField(albumSection, 'Track Art Date') || album.date; - album.coverArtDate = getBasicField(albumSection, 'Cover Art Date') || album.date; - album.dateAdded = getBasicField(albumSection, 'Date Added'); - album.coverArtists = getContributionField(albumSection, 'Cover Art'); - album.hasTrackArt = getBooleanField(albumSection, 'Has Track Art') ?? true; - album.trackCoverArtists = getContributionField(albumSection, 'Track Art'); - album.artTags = getListField(albumSection, 'Art Tags') || []; - album.commentary = getCommentaryField(albumSection); - album.urls = getListField(albumSection, 'URLs') || []; - album.groups = getListField(albumSection, 'Groups') || []; - album.directory = getBasicField(albumSection, 'Directory'); - album.isMajorRelease = getBooleanField(albumSection, 'Major Release') ?? false; - album.isListedOnHomepage = getBooleanField(albumSection, 'Listed on Homepage') ?? true; - - if (album.artists && album.artists.error) { - return {error: `${album.artists.error} (in ${album.name})`}; - } + // String files! For the most part, this is used for translating the + // site to different languages, though you can also customize strings + // for your own 8uild of the site if you'd like. Files here should all + // match the format in strings-default.json in this repository. (If a + // language file is missing any strings, the site code will fall 8ack + // to what's specified in strings-default.json.) + // + // Unlike the other options here, this one's optional - the site will + // 8uild with the default (English) strings if this path is left + // unspecified. + 'lang-path': { + help: `Specify path to language directory, including JSON files that mapping internal string keys to localized language content, and various language metadata\n\nOptional for wiki building, unless the wiki's default language is not English; may be provided via the HSMUSIC_LANG environment variable instead`, + type: 'value', + }, - if (album.coverArtists && album.coverArtists.error) { - return {error: `${album.coverArtists.error} (in ${album.name})`}; - } + 'skip-reference-validation': { + help: `Skips checking and reporting reference errors, which speeds up the build but may silently allow erroneous data to pass through`, + type: 'flag', + }, - if (album.commentary && album.commentary.error) { - return {error: `${album.commentary.error} (in ${album.name})`}; - } + // Thum8nail gener8tion is *usually* something you want, 8ut it can 8e + // kinda a pain to run every time, since it does necessit8te reading + // every media file at run time. Pass this to skip it. + 'skip-thumbs': { + help: `Skip processing and generating thumbnails in media directory (speeds up subsequent builds, but remove this option [or use --thumbs-only] and re-run once when you add or modify media files to ensure thumbnails stay up-to-date!)`, + type: 'flag', + }, - if (album.trackCoverArtists && album.trackCoverArtists.error) { - return {error: `${album.trackCoverArtists.error} (in ${album.name})`}; - } + // Or, if you *only* want to gener8te newly upd8ted thum8nails, you can + // pass this flag! It exits 8efore 8uilding the rest of the site. + 'thumbs-only': { + help: `Skip everything besides processing media directory and generating up-to-date thumbnails (useful when using --skip-thumbs for most runs)`, + type: 'flag', + }, - if (!album.coverArtists) { - return {error: `The album "${album.name}" is missing the "Cover Art" field.`}; - } + 'migrate-thumbs': { + help: `Transfer automatically generated thumbnail files out of an existing media directory and into the easier-to-manage media-cache directory`, + type: 'flag', + }, - album.color = ( - getBasicField(albumSection, 'Color') || - getBasicField(albumSection, 'FG') - ); + 'skip-file-sizes': { + help: `Skips preloading file sizes for images and additional files, which will be left blank in the build`, + type: 'flag', + }, - if (!album.name) { - return {error: `Expected "Album" (name) field!`}; - } + 'skip-media-validation': { + help: `Skips checking and reporting missing and misplaced media files, which isn't necessary if you aren't adding or removing data or updating directories`, + type: 'flag', + }, - if (!album.date) { - return {error: `Expected "Date" field! (in ${album.name})`}; - } + // Just working on data entries and not interested in actually + // generating site HTML yet? This flag will cut execution off right + // 8efore any site 8uilding actually happens. + 'no-build': { + help: `Don't run a build of the site at all; only process data/media and report any errors detected`, + type: 'flag', + }, - if (!album.dateAdded) { - return {error: `Expected "Date Added" field! (in ${album.name})`}; - } + 'no-input': { + help: `Don't wait on input from stdin - assume the device is headless`, + type: 'flag', + }, - if (isNaN(Date.parse(album.date))) { - return {error: `Invalid Date field: "${album.date}" (in ${album.name})`}; - } + 'no-language-reloading': { + help: `Don't reload language files while the build is running\n\nApplied by default for --static-build`, + type: 'flag', + }, - if (isNaN(Date.parse(album.trackArtDate))) { - return {error: `Invalid Track Art Date field: "${album.trackArtDate}" (in ${album.name})`}; - } + 'no-language-reload': {alias: 'no-language-reloading'}, - if (isNaN(Date.parse(album.coverArtDate))) { - return {error: `Invalid Cover Art Date field: "${album.coverArtDate}" (in ${album.name})`}; - } + // Want sweet, sweet trace8ack info in aggreg8te error messages? This + // will print all the juicy details (or at least the first relevant + // line) right to your output, 8ut also pro8a8ly give you a headache + // 8ecause wow that is a lot of visual noise. + 'show-traces': { + help: `Show JavaScript source code paths for reported errors in "aggregate" error displays\n\n(Debugging use only, but please enable this if you're reporting bugs for our issue tracker!)`, + type: 'flag', + }, - if (isNaN(Date.parse(album.dateAdded))) { - return {error: `Invalid Date Added field: "${album.dateAdded}" (in ${album.name})`}; - } + 'show-step-summary': { + help: `Show a summary of all the top-level build steps once hsmusic exits. This is mostly useful for progammer debugging!`, + type: 'flag', + }, - album.date = new Date(album.date); - album.trackArtDate = new Date(album.trackArtDate); - album.coverArtDate = new Date(album.coverArtDate); - album.dateAdded = new Date(album.dateAdded); + 'queue-size': { + help: `Process more or fewer disk files at once to optimize performance or avoid I/O errors, unlimited if set to 0 (between 500 and 700 is usually a safe range for building HSMusic on Windows machines)\nDefaults to ${defaultQueueSize}`, + type: 'value', + validate(size) { + if (parseInt(size) !== parseFloat(size)) return 'an integer'; + if (parseInt(size) < 0) return 'a counting number or zero'; + return true; + }, + }, + queue: {alias: 'queue-size'}, + + 'magick-threads': { + help: `Process more or fewer thumbnail files at once with ImageMagick when generating thumbnails. (Each ImageMagick thread may also make use of multi-core processing at its own utility.)`, + type: 'value', + validate(threads) { + if (parseInt(threads) !== parseFloat(threads)) return 'an integer'; + if (parseInt(threads) < 0) return 'a counting number or zero'; + return true; + } + }, + magick: {alias: 'magick-threads'}, + + // This option is super slow and has the potential for bugs! It puts + // CacheableObject in a mode where every instance is a Proxy which will + // keep track of invalid property accesses. + 'show-invalid-property-accesses': { + help: `Report accesses at runtime to nonexistant properties on wiki data objects, at a dramatic performance cost\n(Internal/development use only)`, + type: 'flag', + }, - if (!album.directory) { - album.directory = getKebabCase(album.name); - } + 'precache-mode': { + help: + `Change the way certain runtime-computed values are preemptively evaluated and cached\n\n` + + `common: Preemptively compute certain properties which are needed for basic data loading and site generation\n\n` + + `all: Compute every visible data property, optimizing rate of content generation, but causing a long stall before the build actually starts\n\n` + + `none: Don't preemptively compute any values - strictly the most efficient, but may result in unpredictably "lopsided" performance for individual steps of loading data and building the site\n\n` + + `Defaults to 'common'`, + type: 'value', + validate(value) { + if (['common', 'all', 'none'].includes(value)) return true; + return 'common, all, or none'; + }, + }, + }; - album.tracks = []; + const cliOptions = await parseOptions(process.argv.slice(2), { + // We don't want to error when we receive these options, so specify them + // here, even though we won't be doing anything with them later. + // (This is a bit of a hack.) + ...buildModeFlagOptions, - // will be overwritten if a group section is found! - album.trackGroups = null; + ...commonOptions, + ...buildOptions, + }); - let group = null; - let trackIndex = 0; + if (cliOptions['help']) { + const indentWrap = (spaces, str) => wrap(str, {width: 60 - spaces, indent: ' '.repeat(spaces)}); - for (const section of sections.slice(1)) { - // Just skip empty sections. Sometimes I paste a 8unch of dividers, - // and this lets the empty sections doing that creates (temporarily) - // exist without raising an error. - if (!section.filter(Boolean).length) { - continue; - } + const showOptions = (msg, options) => { + console.log(colors.bright(msg)); - const groupName = getBasicField(section, 'Group'); - if (groupName) { - group = { - name: groupName, - color: ( - getBasicField(section, 'Color') || - getBasicField(section, 'FG') || - album.color - ), - startIndex: trackIndex, - tracks: [] - }; - if (album.trackGroups) { - album.trackGroups.push(group); - } else { - album.trackGroups = [group]; - } - continue; - } + const entries = Object.entries(options); + const sortedOptions = sortByName(entries + .map(([name, descriptor]) => ({name, descriptor}))); - trackIndex++; - - const track = {}; - - track.name = getBasicField(section, 'Track'); - track.commentary = getCommentaryField(section); - track.lyrics = getMultilineField(section, 'Lyrics'); - track.originalDate = getBasicField(section, 'Original Date'); - track.coverArtDate = getBasicField(section, 'Cover Art Date') || track.originalDate || album.trackArtDate; - track.references = getListField(section, 'References') || []; - track.artists = getContributionField(section, 'Artists') || getContributionField(section, 'Artist'); - track.coverArtists = getContributionField(section, 'Track Art'); - track.artTags = getListField(section, 'Art Tags') || []; - track.contributors = getContributionField(section, 'Contributors') || []; - track.directory = getBasicField(section, 'Directory'); - track.aka = getBasicField(section, 'AKA'); - - if (!track.name) { - return {error: `A track section is missing the "Track" (name) field (in ${album.name}, previous: ${album.tracks[album.tracks.length - 1]?.name}).`}; - } + if (!sortedOptions.length) { + console.log(`(No options available)`) + } - let durationString = getBasicField(section, 'Duration') || '0:00'; - track.duration = getDurationInSeconds(durationString); + let justInsertedPaddingLine = false; - if (track.contributors.error) { - return {error: `${track.contributors.error} (in ${track.name}, ${album.name})`}; + for (const {name, descriptor} of sortedOptions) { + if (descriptor.alias) { + continue; } - if (track.commentary && track.commentary.error) { - return {error: `${track.commentary.error} (in ${track.name}, ${album.name})`}; - } + const aliases = entries + .filter(([_name, {alias}]) => alias === name) + .map(([name]) => name); - if (!track.artists) { - // If an al8um has an artist specified (usually 8ecause it's a solo - // al8um), let tracks inherit that artist. We won't display the - // "8y <artist>" string on the al8um listing. - if (album.artists) { - track.artists = album.artists; - } else { - return {error: `The track "${track.name}" is missing the "Artist" field (in ${album.name}).`}; - } + let wrappedHelp, wrappedHelpLines = 0; + if (descriptor.help) { + wrappedHelp = indentWrap(4, descriptor.help); + wrappedHelpLines = wrappedHelp.split('\n').length; } - if (!track.coverArtists) { - if (getBasicField(section, 'Track Art') !== 'none' && album.hasTrackArt) { - if (album.trackCoverArtists) { - track.coverArtists = album.trackCoverArtists; - } else { - return {error: `The track "${track.name}" is missing the "Track Art" field (in ${album.name}).`}; - } - } + if (wrappedHelpLines > 0 && !justInsertedPaddingLine) { + console.log(''); } - if (track.coverArtists && track.coverArtists.length && track.coverArtists[0] === 'none') { - track.coverArtists = null; - } + console.log(colors.bright(` --` + name) + + (aliases.length + ? ` (or: ${aliases.map(alias => colors.bright(`--` + alias)).join(', ')})` + : '') + + (descriptor.help + ? '' + : colors.dim(' (no help provided)'))); - if (!track.directory) { - track.directory = getKebabCase(track.name); + if (wrappedHelp) { + console.log(wrappedHelp); } - if (track.originalDate) { - if (isNaN(Date.parse(track.originalDate))) { - return {error: `The track "${track.name}"'s has an invalid "Original Date" field: "${track.originalDate}"`}; - } - track.date = new Date(track.originalDate); + if (wrappedHelpLines > 1) { + console.log(''); + justInsertedPaddingLine = true; } else { - track.date = album.date; + justInsertedPaddingLine = false; } + } - track.coverArtDate = new Date(track.coverArtDate); - - const hasURLs = getBooleanField(section, 'Has URLs') ?? true; - - track.urls = hasURLs && (getListField(section, 'URLs') || []).filter(Boolean); - - if (hasURLs && !track.urls.length) { - return {error: `The track "${track.name}" should have at least one URL specified.`}; - } - - // 8ack-reference the al8um o8ject! This is very useful for when - // we're outputting the track pages. - track.album = album; + if (!justInsertedPaddingLine) { + console.log(``); + } + }; - if (group) { - track.color = group.color; - group.tracks.push(track); - } else { - track.color = album.color; - } + console.log( + colors.bright(`hsmusic (aka. Homestuck Music Wiki)\n`) + + `static wiki software cataloguing collaborative creation\n`); - album.tracks.push(track); - } + console.log(indentWrap(0, + `The \`hsmusic\` command provides basic control over all parts of generating user-visible HTML pages and website content/structure from provided data, media, and language directories.\n` + + `\n` + + `CLI options are divided into three groups:\n`)); + console.log(` 1) ` + indentWrap(4, + `Common options: These are shared by all build modes and always have the same essential behavior`).trim()); + console.log(` 2) ` + indentWrap(4, + `Build mode selection: One build mode may be selected (or else the default, --static-build, is used), and it decides which entire set of behavior to use for providing site content to the user`).trim()); + console.log(` 3) ` + indentWrap(4, + `Build options: Each build mode has a set of unique options which customize behavior for that build mode`).trim()); + console.log(``); - return album; -} + showOptions(`Common options`, commonOptions); + showOptions(`Build mode selection`, buildModeFlagOptions); -async function processArtistDataFile(file) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - return {error: `Could not read ${file} (${error.code}).`}; + if (buildOptions) { + showOptions(`Build options for --${selectedBuildModeFlag} (${ + usingDefaultBuildMode ? 'default' : 'selected' + })`, buildOptions); } - const contentLines = contents.split('\n'); - const sections = Array.from(getSections(contentLines)); - - return sections.filter(s => s.filter(Boolean).length).map(section => { - const name = getBasicField(section, 'Artist'); - const urls = (getListField(section, 'URLs') || []).filter(Boolean); - const alias = getBasicField(section, 'Alias'); - const hasAvatar = getBooleanField(section, 'Has Avatar') ?? false; - const note = getMultilineField(section, 'Note'); - let directory = getBasicField(section, 'Directory'); + return true; + } - if (!name) { - return {error: 'Expected "Artist" (name) field!'}; - } + const dataPath = cliOptions['data-path'] || process.env.HSMUSIC_DATA; + const mediaPath = cliOptions['media-path'] || process.env.HSMUSIC_MEDIA; + const langPath = cliOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset! - if (!directory) { - directory = getKebabCase(name); - } + const thumbsOnly = cliOptions['thumbs-only'] ?? false; + const noInput = cliOptions['no-input'] ?? false; - if (alias) { - return {name, directory, alias}; - } else { - return {name, directory, urls, note, hasAvatar}; - } - }); -} + showStepStatusSummary = cliOptions['show-step-summary'] ?? false; -async function processFlashDataFile(file) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - return {error: `Could not read ${file} (${error.code}).`}; - } + const showAggregateTraces = cliOptions['show-traces'] ?? false; - const contentLines = contents.split('\n'); - const sections = Array.from(getSections(contentLines)); - - let act, color; - return sections.map(section => { - if (getBasicField(section, 'ACT')) { - act = getBasicField(section, 'ACT'); - color = ( - getBasicField(section, 'Color') || - getBasicField(section, 'FG') - ); - const anchor = getBasicField(section, 'Anchor'); - const jump = getBasicField(section, 'Jump'); - const jumpColor = getBasicField(section, 'Jump Color') || color; - return {act8r8k: true, name: act, color, anchor, jump, jumpColor}; - } + const precacheMode = cliOptions['precache-mode'] ?? 'common'; + const showInvalidPropertyAccesses = cliOptions['show-invalid-property-accesses'] ?? false; - const name = getBasicField(section, 'Flash'); - let page = getBasicField(section, 'Page'); - let directory = getBasicField(section, 'Directory'); - let date = getBasicField(section, 'Date'); - const jiff = getBasicField(section, 'Jiff'); - const tracks = getListField(section, 'Tracks') || []; - const contributors = getContributionField(section, 'Contributors') || []; - const urls = (getListField(section, 'URLs') || []).filter(Boolean); - - if (!name) { - return {error: 'Expected "Flash" (name) field!'}; - } + // Makes writing nicer on the CPU and file I/O parts of the OS, with a + // marginal performance deficit while waiting for file writes to finish + // before proceeding to more page processing. + const queueSize = +(cliOptions['queue-size'] ?? defaultQueueSize); - if (!page && !directory) { - return {error: 'Expected "Page" or "Directory" field!'}; - } + const magickThreads = +(cliOptions['magick-threads'] ?? defaultMagickThreads); - if (!directory) { - directory = page; - } + if (!dataPath) { + logError`${`Expected --data-path option or HSMUSIC_DATA to be set`}`; + } - if (!date) { - return {error: 'Expected "Date" field!'}; - } + if (!mediaPath) { + logError`${`Expected --media-path option or HSMUSIC_MEDIA to be set`}`; + } - if (isNaN(Date.parse(date))) { - return {error: `Invalid Date field: "${date}"`}; - } + if (!dataPath || !mediaPath) { + return false; + } - date = new Date(date); + if (cliOptions['no-build']) { + logInfo`Won't generate any site or page files this run (--no-build passed).`; - return {name, page, directory, date, contributors, tracks, urls, act, color, jiff}; + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_NOT_APPLICABLE, + annotation: `--no-build provided`, }); -} - -async function processNewsDataFile(file) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - return {error: `Could not read ${file} (${error.code}).`}; + } else { + if (usingDefaultBuildMode) { + logInfo`No build mode specified, will use default: ${selectedBuildModeFlag}`; + } else { + logInfo`Will use specified build mode: ${selectedBuildModeFlag}`; } + } - const contentLines = contents.split('\n'); - const sections = Array.from(getSections(contentLines)); + // Finish setting up defaults by combining information from all options. - return sections.map(section => { - const name = getBasicField(section, 'Name'); - if (!name) { - return {error: 'Expected "Name" field!'}; - } + const _fallbackStep = (stepKey, { + default: defaultValue, - const directory = getBasicField(section, 'Directory') || getBasicField(section, 'ID'); - if (!directory) { - return {error: 'Expected "Directory" field!'}; - } + cli: { + flag: cliFlag = null, + negate: cliFlagNegates = false, + warn: cliFlagWarning = null, + } = {}, - let body = getMultilineField(section, 'Body'); - if (!body) { - return {error: 'Expected "Body" field!'}; - } - - let date = getBasicField(section, 'Date'); - if (!date) { - return {error: 'Expected "Date" field!'}; - } + buildConfig: buildConfigKey, + }) => { + const {[buildConfigKey]: buildConfig} = selectedBuildMode.config; + const {[stepKey]: step} = stepStatusSummary; - if (isNaN(Date.parse(date))) { - return {error: `Invalid date field: "${date}"`}; + if (cliFlag && cliOptions[cliFlag]) { + const cliPart = `--` + cliFlag; + const modePart = `--` + selectedBuildModeFlag; + if (buildConfig?.applicable === false) { + if (cliFlagNegates) { + logWarn`${cliPart} provided, but ${modePart} already skips this step`; + logWarn`Redundant option ${cliPart}`; + } else { + logWarn`${cliPart} provided, but this step isn't applicable for ${modePart}`; + logWarn`Ignoring option ${cliPart}`; } - - date = new Date(date); - - let bodyShort = body.split('<hr class="split">')[0]; - - return { - name, - directory, - body, - bodyShort, - date - }; - }); -} - -async function processTagDataFile(file) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - if (error.code === 'ENOENT') { - return []; + } else if (buildConfig?.required === true) { + if (cliFlagNegates) { + logWarn`${cliPart} provided, but ${modePart} requires this step`; + logWarn`Ignoring option ${cliPart}`; } else { - return {error: `Could not read ${file} (${error.code}).`}; + logWarn`${cliPart} provided, but ${modePart} already requires this step`; + logWarn`Redundant option ${cliPart}`; + } + } else { + if (cliFlagNegates) { + step.status = STATUS_NOT_APPLICABLE; + step.annotation = `--${cliFlag} provided`; } + if (cliFlagWarning) { + for (const line of cliFlagWarning.split('\n')) { + logWarn(line); + } + } + } } - const contentLines = contents.split('\n'); - const sections = Array.from(getSections(contentLines)); - - return sections.map(section => { - let isCW = false; + if (buildConfig?.applicable === false) { + step.status = STATUS_NOT_APPLICABLE; + step.annotation = `N/A for --${selectedBuildModeFlag}`; + return; + } - let name = getBasicField(section, 'Tag'); - if (!name) { - name = getBasicField(section, 'CW'); - isCW = true; - if (!name) { - return {error: 'Expected "Tag" or "CW" field!'}; - } - } + if (buildConfig?.default === 'skip') { + step.status = STATUS_NOT_APPLICABLE; + step.annotation = `default for --${selectedBuildModeFlag}`; + return; + } - let color; - if (!isCW) { - color = getBasicField(section, 'Color'); - if (!color) { - return {error: 'Expected "Color" field!'}; - } + switch (defaultValue) { + case 'skip': + step.status = STATUS_NOT_APPLICABLE; + if (cliFlag && !cliFlagNegates) { + step.annotation = `--${cliFlag} not provided`; } + break; - const directory = getKebabCase(name); - - return { - name, - directory, - isCW, - color - }; - }); -} + case 'perform': + break; -async function processGroupDataFile(file) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - if (error.code === 'ENOENT') { - return []; - } else { - return {error: `Could not read ${file} (${error.code}).`}; - } + default: + throw new Error(`Invalid default step status ${defaultValue}`); } + }; - const contentLines = contents.split('\n'); - const sections = Array.from(getSections(contentLines)); + { + let errored = false; - let category, color; - return sections.map(section => { - if (getBasicField(section, 'Category')) { - category = getBasicField(section, 'Category'); - color = getBasicField(section, 'Color'); - return {isCategory: true, name: category, color}; - } + const fallbackStep = (stepKey, options) => { + try { + _fallbackStep(stepKey, options); + } catch (error) { + logError`Error determining fallback for step ${stepKey}`; + showAggregate(error); + errored = true; + } + }; - const name = getBasicField(section, 'Group'); - if (!name) { - return {error: 'Expected "Group" field!'}; - } + fallbackStep('filterReferenceErrors', { + default: 'perform', + buildConfig: null, + cli: { + flag: 'skip-reference-validation', + negate: true, + warn: + `Skipping reference validation. If any reference errors are present\n` + + `in data, they will be silently passed along to the build.`, + } + }); - let directory = getBasicField(section, 'Directory'); - if (!directory) { - directory = getKebabCase(name); - } + fallbackStep('generateThumbnails', { + default: 'perform', + buildConfig: 'thumbs', + cli: { + flag: 'skip-thumbs', + negate: true, + }, + }); - let description = getMultilineField(section, 'Description'); - if (!description) { - return {error: 'Expected "Description" field!'}; - } + fallbackStep('migrateThumbnails', { + default: 'skip', + buildConfig: null, + cli: { + flag: 'migrate-thumbs', + }, + }); - let descriptionShort = description.split('<hr class="split">')[0]; + fallbackStep('preloadFileSizes', { + default: 'perform', + buildConfig: 'fileSizes', + cli: { + flag: 'skip-file-sizes', + negate: true, + }, + }); - const urls = (getListField(section, 'URLs') || []).filter(Boolean); + fallbackStep('verifyImagePaths', { + default: 'perform', + buildConfig: 'mediaValidation', + cli: { + flag: 'skip-media-validation', + negate: true, + warning: + `Skipping media validation. If any media files are missing or misplaced,\n` + + `those errors will be silently passed along to the build.`, + }, + }); - return { - isGroup: true, - name, - directory, - description, - descriptionShort, - urls, - category, - color - }; + fallbackStep('watchLanguageFiles', { + default: 'perform', + buildConfig: 'languageReloading', + cli: { + flag: 'no-language-reloading', + negate: true, + }, }); -} -async function processStaticPageDataFile(file) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - if (error.code === 'ENOENT') { - return []; - } else { - return {error: `Could not read ${file} (${error.code}).`}; - } + if (errored) { + return false; } + } - const contentLines = contents.split('\n'); - const sections = Array.from(getSections(contentLines)); + if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.loadThumbnailCache, { + status: STATUS_NOT_APPLICABLE, + annotation: `using cache from thumbnail generation`, + }); + } - return sections.map(section => { - const name = getBasicField(section, 'Name'); - if (!name) { - return {error: 'Expected "Name" field!'}; - } + if (stepStatusSummary.watchLanguageFiles.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.loadLanguageFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `watching for changes instead`, + }); + } + + switch (precacheMode) { + case 'common': + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is common, not all`, + }); + + break; + + case 'all': + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is all, not common`, + }); + + break; + + case 'none': + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is none`, + }); + + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_NOT_APPLICABLE, + annotation: `--precache-mode is none`, + }); + + break; + } + + if (!langPath) { + Object.assign(stepStatusSummary.loadLanguageFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `neither --lang-path nor HSMUSIC_LANG provided`, + }); - const shortName = getBasicField(section, 'Short Name') || name; + Object.assign(stepStatusSummary.watchLanguageFiles, { + status: STATUS_NOT_APPLICABLE, + annotation: `neither --lang-path nor HSMUSIC_LANG provided`, + }); + } + + if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_APPLICABLE && thumbsOnly) { + logInfo`Well, you've put yourself rather between a roc and a hard place, hmmmm?`; + return false; + } + + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + const {mediaCachePath, annotation: mediaCachePathAnnotation} = + await determineMediaCachePath({ + mediaPath, + providedMediaCachePath: + cliOptions['media-cache-path'] || process.env.HSMUSIC_MEDIA_CACHE, + disallowDoubling: + stepStatusSummary.migrateThumbnails.status === STATUS_NOT_STARTED, + }); - let directory = getBasicField(section, 'Directory'); - if (!directory) { - return {error: 'Expected "Directory" field!'}; - } + if (!mediaCachePath) { + logError`Couldn't determine a media cache path. (${mediaCachePathAnnotation})`; - let content = getMultilineField(section, 'Content'); - if (!content) { - return {error: 'Expected "Content" field!'}; - } + switch (mediaCachePathAnnotation) { + case 'inferred path does not have cache': + logError`If you're certain this is the right path, you can provide it via`; + logError`${'--media-cache-path'} or ${'HSMUSIC_MEDIA_CACHE'}, and it should work.`; + break; - let stylesheet = getMultilineField(section, 'Style') || ''; + case 'inferred path not readable': + logError`The folder couldn't be read, which usually indicates`; + logError`a permissions error. Try to resolve this, or provide`; + logError`a new path with ${'--media-cache-path'} or ${'HSMUSIC_MEDIA_CACHE'}.`; + break; - let listed = getBooleanField(section, 'Listed') ?? true; + case 'media path not provided': /* unreachable */ + logError`It seems a ${'--media-path'} (or ${'HSMUSIC_MEDIA'}) wasn't provided.`; + logError`Make sure one of these is actually pointing to a path that exists.`; + break; + } - return { - name, - shortName, - directory, - content, - stylesheet, - listed - }; + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_FATAL_ERROR, + annotation: mediaCachePathAnnotation, + timeEnd: Date.now(), }); -} -async function processWikiInfoFile(file) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - return {error: `Could not read ${file} (${error.code}).`}; - } + return false; + } - // Unlike other data files, the site info data file isn't 8roken up into - // more than one entry. So we operate on the plain old contentLines array, - // rather than dividing into sections like we usually do! - const contentLines = contents.split('\n'); + logInfo`Using media cache at: ${mediaCachePath} (${mediaCachePathAnnotation})`; - const name = getBasicField(contentLines, 'Name'); - if (!name) { - return {error: 'Expected "Name" field!'}; - } + Object.assign(stepStatusSummary.determineMediaCachePath, { + status: STATUS_DONE_CLEAN, + annotation: mediaCachePathAnnotation, + timeEnd: Date.now(), + }); - const shortName = getBasicField(contentLines, 'Short Name') || name; - - const color = getBasicField(contentLines, 'Color') || '#0088ff'; - - // This is optional! Without it, <meta rel="canonical"> tags won't 8e - // gener8ted. - const canonicalBase = getBasicField(contentLines, 'Canonical Base'); - - // This is optional! Without it, the site will default to 8uilding in - // English. (This is only really relevant if you've provided string files - // for non-English languages.) - const defaultLanguage = getBasicField(contentLines, 'Default Language'); - - // Also optional! In charge of <meta rel="description">. - const description = getBasicField(contentLines, 'Description'); - - const footer = getMultilineField(contentLines, 'Footer') || ''; - - // We've had a comment lying around for ages, just reading: - // "Might ena8le this later... we'll see! Eventually. May8e." - // We still haven't! 8ut hey, the option's here. - const enableArtistAvatars = getBooleanField(contentLines, 'Enable Artist Avatars') ?? false; - - const enableFlashesAndGames = getBooleanField(contentLines, 'Enable Flashes & Games') ?? false; - const enableListings = getBooleanField(contentLines, 'Enable Listings') ?? false; - const enableNews = getBooleanField(contentLines, 'Enable News') ?? false; - const enableArtTagUI = getBooleanField(contentLines, 'Enable Art Tag UI') ?? false; - const enableGroupUI = getBooleanField(contentLines, 'Enable Group UI') ?? false; - - return { - name, - shortName, - color, - canonicalBase, - defaultLanguage, - description, - footer, - features: { - artistAvatars: enableArtistAvatars, - flashesAndGames: enableFlashesAndGames, - listings: enableListings, - news: enableNews, - artTagUI: enableArtTagUI, - groupUI: enableGroupUI - } - }; -} + if (stepStatusSummary.migrateThumbnails.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.migrateThumbnails, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); -async function processHomepageInfoFile(file) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - return {error: `Could not read ${file} (${error.code}).`}; + const result = await migrateThumbsIntoDedicatedCacheDirectory({ + mediaPath, + mediaCachePath, + queueSize, + }); + + if (result.succses) { + Object.assign(stepStatusSummary.migrateThumbnails, { + status: STATUS_FATAL_ERROR, + annotation: `view log for details`, + timeEnd: Date.now(), + }); + + return false; } - const contentLines = contents.split('\n'); - const sections = Array.from(getSections(contentLines)); + logInfo`Good to go! Run hsmusic again without ${'--migrate-thumbs'} to start`; + logInfo`using the migrated media cache.`; + + Object.assign(stepStatusSummary.migrateThumbnails, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); - const [ firstSection, ...rowSections ] = sections; + return true; + } - const sidebar = getMultilineField(firstSection, 'Sidebar'); + const niceShowAggregate = (error, ...opts) => { + showAggregate(error, { + showTraces: showAggregateTraces, + pathToFileURL: (f) => path.relative(__dirname, fileURLToPath(f)), + ...opts, + }); + }; - const validRowTypes = ['albums']; + if ( + stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED && + stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED + ) { + throw new Error(`Unable to continue with both loadThumbnailCache and generateThumbnails`); + } - const rows = rowSections.map(section => { - const name = getBasicField(section, 'Row'); - if (!name) { - return {error: 'Expected "Row" (name) field!'}; - } + let thumbsCache; - const color = getBasicField(section, 'Color'); + if (stepStatusSummary.loadThumbnailCache.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.loadThumbnailCache, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - const type = getBasicField(section, 'Type'); - if (!type) { - return {error: 'Expected "Type" field!'}; - } + const thumbsCachePath = path.join(mediaCachePath, thumbsCacheFile); - if (!validRowTypes.includes(type)) { - return {error: `Expected "Type" field to be one of: ${validRowTypes.join(', ')}`}; - } + try { + thumbsCache = JSON.parse(await readFile(thumbsCachePath)); + } catch (error) { + if (error.code === 'ENOENT') { + logError`The thumbnail cache doesn't exist, and it's necessary to build` + logError`the website. Please run once without ${'--skip-thumbs'} - after` + logError`that you'll be good to go and don't need to process thumbnails` + logError`again!`; + + Object.assign(stepStatusSummary.loadThumbnailCache, { + status: STATUS_FATAL_ERROR, + annotation: `cache does not exist`, + timeEnd: Date.now(), + }); - const row = {name, color, type}; + return false; + } else { + logError`Malformed or unreadable thumbnail cache file: ${error}`; + logError`Path: ${thumbsCachePath}`; + logError`The thumbbnail cache is necessary to build the site, so you'll`; + logError`have to investigate this to get the build working. Try running`; + logError`again without ${'--skip-thumbs'}. If you can't get it working,`; + logError`you're welcome to message in the HSMusic Discord and we'll try`; + logError`to help you out with troubleshooting!`; + logError`${'https://hsmusic.wiki/discord/'}`; + + Object.assign(stepStatusSummary.loadThumbnailCache, { + status: STATUS_FATAL_ERROR, + annotation: `cache malformed or unreadable`, + timeEnd: Date.now(), + }); - switch (type) { - case 'albums': { - const group = getBasicField(section, 'Group') || null; - const albums = getListField(section, 'Albums') || []; + return false; + } + } - if (!group && !albums) { - return {error: 'Expected "Group" and/or "Albums" field!'}; - } + logInfo`Thumbnail cache file successfully read.`; - let groupCount = getBasicField(section, 'Count'); - if (group && !groupCount) { - return {error: 'Expected "Count" field!'}; - } + Object.assign(stepStatusSummary.loadThumbnailCache, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); - if (groupCount) { - if (isNaN(parseInt(groupCount))) { - return {error: `Invalid Count field: "${groupCount}"`}; - } + logInfo`Skipping thumbnail generation.`; + } else if (stepStatusSummary.generateThumbnails.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.generateThumbnails, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - groupCount = parseInt(groupCount); - } + logInfo`Begin thumbnail generation... -----+`; - const actions = getListField(section, 'Actions') || []; + const result = await genThumbs({ + mediaPath, + mediaCachePath, - return {...row, group, groupCount, albums, actions}; - } - } + queueSize, + magickThreads, + quiet: !thumbsOnly, }); - return {sidebar, rows}; -} + logInfo`Done thumbnail generation! --------+`; -function getDurationInSeconds(string) { - const parts = string.split(':').map(n => parseInt(n)) - if (parts.length === 3) { - return parts[0] * 3600 + parts[1] * 60 + parts[2] - } else if (parts.length === 2) { - return parts[0] * 60 + parts[1] - } else { - return 0 + if (!result.success) { + Object.assign(stepStatusSummary.generateThumbnails, { + status: STATUS_FATAL_ERROR, + annotation: `view log for details`, + timeEnd: Date.now(), + }); + + return false; } -} -const stringifyIndent = 0; + Object.assign(stepStatusSummary.generateThumbnails, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); -const toRefs = (label, objectOrArray) => { - if (Array.isArray(objectOrArray)) { - return objectOrArray.filter(Boolean).map(x => `${label}:${x.directory}`); - } else if (objectOrArray.directory) { - throw new Error('toRefs should not be passed a single object with directory'); - } else if (typeof objectOrArray === 'object') { - return Object.fromEntries(Object.entries(objectOrArray) - .map(([ key, value ]) => [key, toRefs(key, value)])); - } else { - throw new Error('toRefs should be passed an array or object of arrays'); - } -}; - -function stringifyRefs(key, value) { - switch (key) { - case 'tracks': - case 'references': - case 'referencedBy': - return toRefs('track', value); - case 'artists': - case 'contributors': - case 'coverArtists': - case 'trackCoverArtists': - return value && value.map(({ who, what }) => ({who: `artist:${who.directory}`, what})); - case 'albums': return toRefs('album', value); - case 'flashes': return toRefs('flash', value); - case 'groups': return toRefs('group', value); - case 'artTags': return toRefs('tag', value); - case 'aka': return value && `track:${value.directory}`; - default: - return value; + if (thumbsOnly) { + return true; } -} -function stringifyAlbumData({wikiData}) { - return JSON.stringify(wikiData.albumData, (key, value) => { - switch (key) { - case 'commentary': - return ''; - default: - return stringifyRefs(key, value); - } - }, stringifyIndent); -} + thumbsCache = result.cache; + } else { + thumbsCache = {}; + } -function stringifyTrackData({wikiData}) { - return JSON.stringify(wikiData.trackData, (key, value) => { - switch (key) { - case 'album': - case 'commentary': - case 'otherReleases': - return undefined; - default: - return stringifyRefs(key, value); - } - }, stringifyIndent); -} - -function stringifyFlashData({wikiData}) { - return JSON.stringify(wikiData.flashData, (key, value) => { - switch (key) { - case 'act': - case 'commentary': - return undefined; - default: - return stringifyRefs(key, value); - } - }, stringifyIndent); -} + if (showInvalidPropertyAccesses) { + CacheableObject.DEBUG_SLOW_TRACK_INVALID_PROPERTIES = true; + } -function stringifyArtistData({wikiData}) { - return JSON.stringify(wikiData.artistData, (key, value) => { - switch (key) { - case 'asAny': - return; - case 'asArtist': - case 'asContributor': - case 'asCoverArtist': - return toRefs('track', value); - default: - return stringifyRefs(key, value); - } - }, stringifyIndent); -} + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); -function img({ - src, - alt, - thumb: thumbKey, - reveal, - id, - class: className, - width, - height, - link = false, - lazy = false, - square = false -}) { - const willSquare = square; - const willLink = typeof link === 'string' || link; - - const originalSrc = src; - const thumbSrc = thumbKey ? thumb[thumbKey](src) : src; - - const imgAttributes = html.attributes({ - id: link ? '' : id, - class: className, - alt, - width, - height - }); + let processDataAggregate, wikiDataResult; - const nonlazyHTML = wrap(`<img src="${thumbSrc}" ${imgAttributes}>`); - const lazyHTML = lazy && wrap(`<img class="lazy" data-original="${thumbSrc}" ${imgAttributes}>`, true); + try { + ({aggregate: processDataAggregate, result: wikiDataResult} = + await loadAndProcessDataDocuments({dataPath})); + } catch (error) { + console.error(error); - if (lazy) { - return fixWS` - <noscript>${nonlazyHTML}</noscript> - ${lazyHTML} - `; - } else { - return nonlazyHTML; - } + logError`There was a JavaScript error loading data files.`; + fileIssue(); - function wrap(input, hide = false) { - let wrapped = input; + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_FATAL_ERROR, + annotation: `javascript error - view log for details`, + timeEnd: Date.now(), + }); - wrapped = `<div class="image-inner-area">${wrapped}</div>`; - wrapped = `<div class="image-container">${wrapped}</div>`; + return false; + } - if (reveal) { - wrapped = fixWS` - <div class="reveal"> - ${wrapped} - <span class="reveal-text">${reveal}</span> - </div> - `; - } + Object.assign(wikiData, wikiDataResult); - if (willSquare) { - wrapped = html.tag('div', {class: 'square-content'}, wrapped); - wrapped = html.tag('div', {class: ['square', hide && !willLink && 'js-hide']}, wrapped); - } + { + const logThings = (prop, label) => { + const array = + (Array.isArray(prop) + ? prop + : wikiData[prop]); - if (willLink) { - wrapped = html.tag('a', { - id, - class: ['box', hide && 'js-hide'], - href: typeof link === 'string' ? link : originalSrc - }, wrapped); - } + logInfo` - ${array?.length ?? colors.red('(Missing!)')} ${colors.normal(colors.dim(label))}`; + } - return wrapped; + try { + logInfo`Loaded data and processed objects:`; + logThings('albumData', 'albums'); + logThings('trackData', 'tracks'); + logThings(wikiData.artistData.filter(artist => !artist.isAlias), 'artists'); + if (wikiData.flashData) { + logThings('flashData', 'flashes'); + logThings('flashActData', 'flash acts'); + logThings('flashSideData', 'flash sides'); + } + logThings('groupData', 'groups'); + logThings('groupCategoryData', 'group categories'); + logThings('artTagData', 'art tags'); + if (wikiData.newsData) { + logThings('newsData', 'news entries'); + } + logThings('staticPageData', 'static pages'); + if (wikiData.homepageLayout) { + logInfo` - ${1} homepage layout (${ + wikiData.homepageLayout.rows.length + } rows)`; + } + if (wikiData.wikiInfo) { + logInfo` - ${1} wiki config file`; + } + } catch (error) { + console.error(`Error showing data summary:`, error); } -} -function validateWritePath(path, urlGroup) { - if (!Array.isArray(path)) { - return {error: `Expected array, got ${path}`}; + let errorless = true; + try { + processDataAggregate.close(); + } catch (error) { + niceShowAggregate(error); + logWarn`The above errors were detected while processing data files.`; + errorless = false; } - const { paths } = urlGroup; + if (!wikiData.wikiInfo) { + logError`Can't proceed without wiki info file successfully loading`; - const definedKeys = Object.keys(paths); - const specifiedKey = path[0]; + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_FATAL_ERROR, + annotation: `wiki info object not available`, + timeEnd: Date.now(), + }); - if (!definedKeys.includes(specifiedKey)) { - return {error: `Specified key ${specifiedKey} isn't defined`}; + return false; } - const expectedArgs = paths[specifiedKey].match(/<>/g)?.length ?? 0; - const specifiedArgs = path.length - 1; + if (errorless) { + logInfo`All data files processed without any errors - nice!`; - if (specifiedArgs !== expectedArgs) { - return {error: `Expected ${expectedArgs} arguments, got ${specifiedArgs}`}; - } + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } else { + logWarn`If the remaining valid data is complete enough, the wiki will`; + logWarn`still build - but all errored data will be skipped.`; + logWarn`(Resolve errors for more complete output!)`; + + Object.assign(stepStatusSummary.loadDataFiles, { + status: STATUS_HAS_WARNINGS, + annotation: `view log for details`, + timeEnd: Date.now(), + }); + } + } + + // Link data arrays so that all essential references between objects are + // complete, so properties (like dates!) are inherited where that's + // appropriate. + + Object.assign(stepStatusSummary.linkWikiDataArrays, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + linkWikiDataArrays(wikiData); + + Object.assign(stepStatusSummary.linkWikiDataArrays, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + + if (precacheMode === 'common') { + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - return {success: true}; -} + const commonDataMap = { + albumData: new Set([ + // Needed for sorting + 'date', 'tracks', + // Needed for computing page paths + 'aliasedArtist', 'commentary', 'coverArtistContribs', + ]), + + artTagData: new Set([ + // Needed for computing page paths + 'isContentWarning', + ]), + + flashData: new Set([ + // Needed for sorting + 'act', 'date', + ]), + + flashActData: new Set([ + // Needed for sorting + 'flashes', + ]), + + groupData: new Set([ + // Needed for computing page paths + 'albums', + ]), + + listingSpec: new Set([ + // Needed for computing page paths + 'contentFunction', 'featureFlag', + ]), + + trackData: new Set([ + // Needed for sorting + 'album', 'date', + // Needed for computing page paths + 'commentary', 'coverArtistContribs', + ]), + }; -function validateWriteObject(obj) { - if (typeof obj !== 'object') { - return {error: `Expected object, got ${typeof obj}`}; + for (const [wikiDataKey, properties] of Object.entries(commonDataMap)) { + const thingData = wikiData[wikiDataKey]; + const allProperties = new Set(['name', 'directory', ...properties]); + for (const thing of thingData) { + for (const property of allProperties) { + void thing[property]; + } + } } - if (typeof obj.type !== 'string') { - return {error: `Expected type to be string, got ${obj.type}`}; - } + Object.assign(stepStatusSummary.precacheCommonData, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } - switch (obj.type) { - case 'legacy': { - if (typeof obj.write !== 'function') { - return {error: `Expected write to be string, got ${obj.write}`}; - } + // Filter out any things with duplicate directories throughout the data, + // warning about them too. - break; - } + Object.assign(stepStatusSummary.reportDuplicateDirectories, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - case 'page': { - const path = validateWritePath(obj.path, urlSpec.localized); - if (path.error) { - return {error: `Path validation failed: ${path.error}`}; - } + try { + reportDuplicateDirectories(wikiData, {getAllFindSpecs}); + logInfo`No duplicate directories found - nice!`; - if (typeof obj.page !== 'function') { - return {error: `Expected page to be function, got ${obj.content}`}; - } + Object.assign(stepStatusSummary.reportDuplicateDirectories, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } catch (aggregate) { + niceShowAggregate(aggregate); + + logWarn`The above duplicate directories were detected while reviewing data files.`; + logWarn`Since it's impossible to automatically determine which one's directory is`; + logWarn`correct, the build can't continue. Specify unique 'Directory' fields in`; + logWarn`some or all of these data entries to resolve the errors.`; + + Object.assign(stepStatusSummary.reportDuplicateDirectories, { + status: STATUS_FATAL_ERROR, + annotation: `duplicate directories found`, + timeEnd: Date.now(), + }); - break; - } + return false; + } - case 'data': { - const path = validateWritePath(obj.path, urlSpec.data); - if (path.error) { - return {error: `Path validation failed: ${path.error}`}; - } + // Filter out any reference errors throughout the data, warning about them + // too. - if (typeof obj.data !== 'function') { - return {error: `Expected data to be function, got ${obj.data}`}; - } + if (stepStatusSummary.filterReferenceErrors.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.filterReferenceErrors, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - break; - } + const filterReferenceErrorsAggregate = + filterReferenceErrors(wikiData, {bindFind}); - case 'redirect': { - const fromPath = validateWritePath(obj.fromPath, urlSpec.localized); - if (fromPath.error) { - return {error: `Path (fromPath) validation failed: ${fromPath.error}`}; - } + try { + filterReferenceErrorsAggregate.close(); - const toPath = validateWritePath(obj.toPath, urlSpec.localized); - if (toPath.error) { - return {error: `Path (toPath) validation failed: ${toPath.error}`}; - } + logInfo`All references validated without any errors - nice!`; - if (typeof obj.title !== 'function') { - return {error: `Expected title to be function, got ${obj.title}`}; - } + Object.assign(stepStatusSummary.filterReferenceErrors, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } catch (error) { + niceShowAggregate(error); - break; - } + logWarn`The above errors were detected while validating references in data files.`; + logWarn`The wiki will still build, but these connections between data objects`; + logWarn`will be completely skipped. Resolve the errors for more complete output.`; - default: { - return {error: `Unknown type: ${obj.type}`}; - } + Object.assign(stepStatusSummary.filterReferenceErrors, { + status: STATUS_HAS_WARNINGS, + annotation: `view log for details`, + timeEnd: Date.now(), + }); } + } - return {success: true}; -} + if (stepStatusSummary.reportContentTextErrors.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.reportContentTextErrors, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); -async function writeData(subKey, directory, data) { - const paths = writePage.paths('', 'data.' + subKey, directory, {file: 'data.json'}); - await writePage.write(JSON.stringify(data), {paths}); -} + try { + reportContentTextErrors(wikiData, {bindFind}); + logInfo`All content text validated without any errors - nice!`; -// This used to 8e a function! It's long 8een divided into multiple helper -// functions, and nowadays we just directly access those, rather than ever -// touching the original one (which had contained everything). -const writePage = {}; - -writePage.to = ({ - baseDirectory, - pageSubKey, - paths -}) => (targetFullKey, ...args) => { - const [ groupKey, subKey ] = targetFullKey.split('.'); - let path = paths.subdirectoryPrefix; - // When linking to *outside* the localized area of the site, we need to - // make sure the result is correctly relative to the 8ase directory. - if (groupKey !== 'localized' && baseDirectory) { - path += urls.from('localizedWithBaseDirectory.' + pageSubKey).to(targetFullKey, ...args); - } else { - // If we're linking inside the localized area (or there just is no - // 8ase directory), the 8ase directory doesn't matter. - path += urls.from('localized.' + pageSubKey).to(targetFullKey, ...args); - } - return path; -}; - -writePage.html = (pageFn, { - paths, - strings, - to, - transformMultiline, - wikiData -}) => { - const { wikiInfo } = wikiData; - - let { - title = '', - meta = {}, - theme = '', - stylesheet = '', - - // missing properties are auto-filled, see below! - body = {}, - banner = {}, - main = {}, - sidebarLeft = {}, - sidebarRight = {}, - nav = {}, - footer = {} - } = pageFn({to}); - - body.style ??= ''; - - theme = theme || getThemeString(wikiInfo.color); - - banner ||= {}; - banner.classes ??= []; - banner.src ??= ''; - banner.position ??= ''; - banner.dimensions ??= [0, 0]; - - main.classes ??= []; - main.content ??= ''; - - sidebarLeft ??= {}; - sidebarRight ??= {}; - - for (const sidebar of [sidebarLeft, sidebarRight]) { - sidebar.classes ??= []; - sidebar.content ??= ''; - sidebar.collapse ??= true; - } + Object.assign(stepStatusSummary.reportContentTextErrors, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } catch (error) { + niceShowAggregate(error); + + logWarn`The above errors were detected while processing content text in data files.`; + logWarn`The wiki will still build, but placeholders will be displayed in these spots.`; + logWarn`Resolve the errors for more complete output.`; - nav.classes ??= []; - nav.content ??= ''; - nav.links ??= []; - - footer.classes ??= []; - footer.content ??= (wikiInfo.footer ? transformMultiline(wikiInfo.footer) : ''); - - const canonical = (wikiInfo.canonicalBase - ? wikiInfo.canonicalBase + paths.pathname - : ''); - - const collapseSidebars = (sidebarLeft.collapse !== false) && (sidebarRight.collapse !== false); - - const mainHTML = main.content && html.tag('main', { - id: 'content', - class: main.classes - }, main.content); - - const footerHTML = footer.content && html.tag('footer', { - id: 'footer', - class: footer.classes - }, footer.content); - - const generateSidebarHTML = (id, { - content, - multiple, - classes, - collapse = true, - wide = false - }) => (content - ? html.tag('div', - {id, class: [ - 'sidebar-column', - 'sidebar', - wide && 'wide', - !collapse && 'no-hide', - ...classes - ]}, - content) - : multiple ? html.tag('div', - {id, class: [ - 'sidebar-column', - 'sidebar-multiple', - wide && 'wide', - !collapse && 'no-hide' - ]}, - multiple.map(content => html.tag('div', - {class: ['sidebar', ...classes]}, - content))) - : ''); - - const sidebarLeftHTML = generateSidebarHTML('sidebar-left', sidebarLeft); - const sidebarRightHTML = generateSidebarHTML('sidebar-right', sidebarRight); - - if (nav.simple) { - nav.links = [ - {toHome: true}, - {toCurrentPage: true} - ]; + Object.assign(stepStatusSummary.reportContentTextErrors, { + status: STATUS_HAS_WARNINGS, + annotation: `view log for details`, + timeEnd: Date.now(), + }); } + } - const links = (nav.links || []).filter(Boolean); + // Sort data arrays so that they're all in order! This may use properties + // which are only available after the initial linking. - const navLinkParts = []; - for (let i = 0; i < links.length; i++) { - let cur = links[i]; - const prev = links[i - 1]; - const next = links[i + 1]; + Object.assign(stepStatusSummary.sortWikiDataArrays, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - let { title: linkTitle } = cur; + sortWikiDataArrays(wikiData); - if (cur.toHome) { - linkTitle ??= wikiInfo.shortName; - } else if (cur.toCurrentPage) { - linkTitle ??= title; - } + Object.assign(stepStatusSummary.sortWikiDataArrays, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); - let part = prev && (cur.divider ?? true) ? '/ ' : ''; + if (precacheMode === 'all') { + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - if (typeof cur.html === 'string') { - if (!cur.html) { - logWarn`Empty HTML in nav link ${JSON.stringify(cur)}`; - } - part += `<span>${cur.html}</span>`; - } else { - const attributes = { - class: (cur.toCurrentPage || i === links.length - 1) && 'current', - href: ( - cur.toCurrentPage ? '' : - cur.toHome ? to('localized.home') : - cur.path ? to(...cur.path) : - cur.href ? call(() => { - logWarn`Using legacy href format nav link in ${paths.pathname}`; - return cur.href; - }) : - null) - }; - if (attributes.href === null) { - throw new Error(`Expected some href specifier for link to ${linkTitle} (${JSON.stringify(cur)})`); - } - part += html.tag('a', attributes, linkTitle); - } - navLinkParts.push(part); - } + // TODO: Aggregate errors here, instead of just throwing. + progressCallAll('Caching all data values', Object.entries(wikiData) + .filter(([key]) => + key !== 'listingSpec' && + key !== 'listingTargetSpec') + .map(([key, value]) => + key === 'wikiInfo' ? [key, [value]] : + key === 'homepageLayout' ? [key, [value]] : + [key, value]) + .flatMap(([_key, things]) => things) + .map(thing => () => CacheableObject.cacheAllExposedProperties(thing))); + + Object.assign(stepStatusSummary.precacheAllData, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } - const navHTML = html.tag('nav', { - [html.onlyIfContent]: true, - id: 'header', - class: nav.classes - }, [ - links.length && html.tag('h2', {class: 'highlight-last-link'}, navLinkParts), - nav.content - ]); - - const bannerSrc = ( - banner.src ? banner.src : - banner.path ? to(...banner.path) : - null); - - const bannerHTML = banner.position && bannerSrc && html.tag('div', - { - id: 'banner', - class: banner.classes - }, - html.tag('img', { - src: bannerSrc, - alt: banner.alt, - width: banner.dimensions[0] || 1100, - height: banner.dimensions[1] || 200 - }) - ); - - const layoutHTML = [ - navHTML, - banner.position === 'top' && bannerHTML, - html.tag('div', - {class: ['layout-columns', !collapseSidebars && 'vertical-when-thin']}, - [ - sidebarLeftHTML, - mainHTML, - sidebarRightHTML - ]), - banner.position === 'bottom' && bannerHTML, - footerHTML - ].filter(Boolean).join('\n'); - - const infoCardHTML = fixWS` - <div id="info-card-container"> - <div class="info-card-decor"> - <div class="info-card"> - <div class="info-card-art-container no-reveal"> - ${img({ - class: 'info-card-art', - src: '', - link: true, - square: true - })} - </div> - <div class="info-card-art-container reveal"> - ${img({ - class: 'info-card-art', - src: '', - link: true, - square: true, - reveal: getRevealStringFromWarnings('<span class="info-card-art-warnings"></span>', {strings}) - })} - </div> - <h1 class="info-card-name"><a></a></h1> - <p class="info-card-album">${strings('releaseInfo.from', {album: '<a></a>'})}</p> - <p class="info-card-artists">${strings('releaseInfo.by', {artists: '<span></span>'})}</p> - <p class="info-card-cover-artists">${strings('releaseInfo.coverArtBy', {artists: '<span></span>'})}</p> - </div> - </div> - </div> - `; - - return filterEmptyLines(fixWS` - <!DOCTYPE html> - <html ${html.attributes({ - lang: strings.code, - 'data-rebase-localized': to('localized.root'), - 'data-rebase-shared': to('shared.root'), - 'data-rebase-media': to('media.root'), - 'data-rebase-data': to('data.root') - })}> - <head> - <title>${title}</title> - <meta charset="utf-8"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - ${Object.entries(meta).filter(([ key, value ]) => value).map(([ key, value ]) => `<meta ${key}="${html.escapeAttributeValue(value)}">`).join('\n')} - ${canonical && `<link rel="canonical" href="${canonical}">`} - <link rel="stylesheet" href="${to('shared.staticFile', `site.css?${CACHEBUST}`)}"> - ${(theme || stylesheet) && fixWS` - <style> - ${theme} - ${stylesheet} - </style> - `} - <script src="${to('shared.staticFile', `lazy-loading.js?${CACHEBUST}`)}"></script> - </head> - <body ${html.attributes({style: body.style || ''})}> - <div id="page-container"> - ${mainHTML && fixWS` - <div id="skippers"> - ${[ - ['#content', strings('misc.skippers.skipToContent')], - sidebarLeftHTML && ['#sidebar-left', (sidebarRightHTML - ? strings('misc.skippers.skipToSidebar.left') - : strings('misc.skippers.skipToSidebar'))], - sidebarRightHTML && ['#sidebar-right', (sidebarLeftHTML - ? strings('misc.skippers.skipToSidebar.right') - : strings('misc.skippers.skipToSidebar'))], - footerHTML && ['#footer', strings('misc.skippers.skipToFooter')] - ].filter(Boolean).map(([ href, title ]) => fixWS` - <span class="skipper"><a href="${href}">${title}</a></span> - `).join('\n')} - </div> - `} - ${layoutHTML} - </div> - ${infoCardHTML} - <script type="module" src="${to('shared.staticFile', `client.js?${CACHEBUST}`)}"></script> - </body> - </html> - `); -}; - -writePage.write = async (content, {paths}) => { - await mkdir(paths.outputDirectory, {recursive: true}); - await writeFile(paths.outputFile, content); -}; - -// TODO: This only supports one <>-style argument. -writePage.paths = (baseDirectory, fullKey, directory = '', { - file = 'index.html' -} = {}) => { - const [ groupKey, subKey ] = fullKey.split('.'); - - const pathname = (groupKey === 'localized' && baseDirectory - ? urls.from('shared.root').to('localizedWithBaseDirectory.' + subKey, baseDirectory, directory) - : urls.from('shared.root').to(fullKey, directory)); - - // Needed for the rare directory which itself contains a slash, e.g. for - // listings, with directories like 'albums/by-name'. - const subdirectoryPrefix = '../'.repeat(directory.split('/').length - 1); - - const outputDirectory = path.join(outputPath, pathname); - const outputFile = path.join(outputDirectory, file); - - return { - pathname, - subdirectoryPrefix, - outputDirectory, outputFile - }; -}; - -function writeSymlinks() { - return progressPromiseAll('Writing site symlinks.', [ - link(path.join(__dirname, UTILITY_DIRECTORY), 'shared.utilityRoot'), - link(path.join(__dirname, STATIC_DIRECTORY), 'shared.staticRoot'), - link(mediaPath, 'media.root') - ]); - - async function link(directory, urlKey) { - const pathname = urls.from('shared.root').to(urlKey); - const file = path.join(outputPath, pathname); - try { - await unlink(file); - } catch (error) { - if (error.code !== 'ENOENT') { - throw error; - } - } - await symlink(path.resolve(directory), file); + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { + displayCompositeCacheAnalysis(); + + if (precacheMode === 'all') { + return true; } -} + } -function writeSharedFilesAndPages({strings, wikiData}) { - const { groupData, wikiInfo } = wikiData; + const languageReloading = + stepStatusSummary.watchLanguageFiles.status === STATUS_NOT_STARTED; - const redirect = async (title, from, urlKey, directory) => { - const target = path.relative(from, urls.from('shared.root').to(urlKey, directory)); - const content = generateRedirectPage(title, target, {strings}); - await mkdir(path.join(outputPath, from), {recursive: true}); - await writeFile(path.join(outputPath, from, 'index.html'), content); - }; + Object.assign(stepStatusSummary.loadInternalDefaultLanguage, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - return progressPromiseAll(`Writing files & pages shared across languages.`, [ - groupData?.some(group => group.directory === 'fandom') && - redirect('Fandom - Gallery', 'albums/fandom', 'localized.groupGallery', 'fandom'), + let internalDefaultLanguage; + let internalDefaultLanguageWatcher; - groupData?.some(group => group.directory === 'official') && - redirect('Official - Gallery', 'albums/official', 'localized.groupGallery', 'official'), + let errorLoadingInternalDefaultLanguage = false; - wikiInfo.features.listings && - redirect('Album Commentary', 'list/all-commentary', 'localized.commentaryIndex', ''), + if (languageReloading) { + internalDefaultLanguageWatcher = watchLanguageFile(internalDefaultStringsFile); - writeFile(path.join(outputPath, 'data.json'), fixWS` - { - "albumData": ${stringifyAlbumData({wikiData})}, - ${wikiInfo.features.flashesAndGames && `"flashData": ${stringifyFlashData({wikiData})},`} - "artistData": ${stringifyArtistData({wikiData})} - } - `) - ].filter(Boolean)); -} + try { + await new Promise((resolve, reject) => { + const watcher = internalDefaultLanguageWatcher; -function generateRedirectPage(title, target, {strings}) { - return fixWS` - <!DOCTYPE html> - <html> - <head> - <title>${strings('redirectPage.title', {title})}</title> - <meta charset="utf-8"> - <meta http-equiv="refresh" content="0;url=${target}"> - <link rel="canonical" href="${target}"> - <link rel="stylesheet" href="static/site-basic.css"> - </head> - <body> - <main> - <h1>${strings('redirectPage.title', {title})}</h1> - <p>${strings('redirectPage.infoLine', { - target: `<a href="${target}">${target}</a>` - })}</p> - </main> - </body> - </html> - `; -} + const onReady = () => { + watcher.removeListener('ready', onReady); + watcher.removeListener('error', onError); + resolve(); + }; -// RIP toAnythingMan (previously getHrefOfAnythingMan), 2020-05-25<>2021-05-14. -// ........Yet the function 8reathes life anew as linkAnythingMan! ::::) -function linkAnythingMan(anythingMan, {link, wikiData, ...opts}) { - return ( - wikiData.albumData.includes(anythingMan) ? link.album(anythingMan, opts) : - wikiData.trackData.includes(anythingMan) ? link.track(anythingMan, opts) : - wikiData.flashData?.includes(anythingMan) ? link.flash(anythingMan, opts) : - 'idk bud' - ) -} + const onError = error => { + watcher.removeListener('ready', onReady); + watcher.removeListener('error', onError); + watcher.close(); + reject(error); + }; -async function processLanguageFile(file, defaultStrings = null) { - let contents; - try { - contents = await readFile(file, 'utf-8'); - } catch (error) { - return {error: `Could not read ${file} (${error.code}).`}; + watcher.on('ready', onReady); + watcher.on('error', onError); + }); + + internalDefaultLanguage = internalDefaultLanguageWatcher.language; + } catch (_error) { + // No need to display the error here - it's already printed by + // watchLanguageFile. + errorLoadingInternalDefaultLanguage = true; } + } else { + internalDefaultLanguageWatcher = null; - let json; try { - json = JSON.parse(contents); + internalDefaultLanguage = await processLanguageFile(internalDefaultStringsFile); } catch (error) { - return {error: `Could not parse JSON from ${file} (${error}).`}; + niceShowAggregate(error); + errorLoadingInternalDefaultLanguage = true; } + } - return genStrings(json, { - he, - defaultJSON: defaultStrings?.json, - bindUtilities: { - count, - list - } + if (errorLoadingInternalDefaultLanguage) { + logError`There was an error reading the internal language file.`; + fileIssue(); + + Object.assign(stepStatusSummary.loadInternalDefaultLanguage, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), }); -} -// Wrapper function for running a function once for all languages. -async function wrapLanguages(fn, {writeOneLanguage = null}) { - const k = writeOneLanguage; - const languagesToRun = (k - ? {[k]: languages[k]} - : languages); + return false; + } - const entries = Object.entries(languagesToRun) - .filter(([ key ]) => key !== 'default'); + if (languageReloading) { + // Bypass node.js special-case handling for uncaught error events + internalDefaultLanguageWatcher.on('error', () => {}); + } - for (let i = 0; i < entries.length; i++) { - const [ key, strings ] = entries[i]; + Object.assign(stepStatusSummary.loadInternalDefaultLanguage, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); - const baseDirectory = (strings === languages.default ? '' : strings.code); + let customLanguageWatchers; + let languages; - await fn({ - baseDirectory, - strings - }, i, entries); + if (langPath) { + if (languageReloading) { + Object.assign(stepStatusSummary.watchLanguageFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + } else { + Object.assign(stepStatusSummary.loadLanguageFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); } -} -async function main() { - Error.stackTraceLimit = Infinity; - - const WD = wikiData; - - WD.listingSpec = listingSpec; - WD.listingTargetSpec = listingTargetSpec; - - const miscOptions = await parseOptions(process.argv.slice(2), { - // Data files for the site, including flash, artist, and al8um data, - // and like a jillion other things too. Pretty much everything which - // makes an individual wiki what it is goes here! - 'data-path': { - type: 'value' - }, - - // Static media will 8e referenced in the site here! The contents are - // categorized; check out MEDIA_ALBUM_ART_DIRECTORY and other constants - // near the top of this file (upd8.js). - 'media-path': { - type: 'value' - }, - - // String files! For the most part, this is used for translating the - // site to different languages, though you can also customize strings - // for your own 8uild of the site if you'd like. Files here should all - // match the format in strings-default.json in this repository. (If a - // language file is missing any strings, the site code will fall 8ack - // to what's specified in strings-default.json.) - // - // Unlike the other options here, this one's optional - the site will - // 8uild with the default (English) strings if this path is left - // unspecified. - 'lang-path': { - type: 'value' - }, - - // This is the output directory. It's the one you'll upload online with - // rsync or whatever when you're pushing an upd8, and also the one - // you'd archive if you wanted to make a 8ackup of the whole dang - // site. Just keep in mind that the gener8ted result will contain a - // couple symlinked directories, so if you're uploading, you're pro8a8ly - // gonna want to resolve those yourself. - 'out-path': { - type: 'value' - }, - - // Thum8nail gener8tion is *usually* something you want, 8ut it can 8e - // kinda a pain to run every time, since it does necessit8te reading - // every media file at run time. Pass this to skip it. - 'skip-thumbs': { - type: 'flag' - }, - - // Or, if you *only* want to gener8te newly upd8ted thum8nails, you can - // pass this flag! It exits 8efore 8uilding the rest of the site. - 'thumbs-only': { - type: 'flag' - }, - - // Only want 8uild one language during testing? This can chop down - // 8uild times a pretty 8ig chunk! Just pass a single language code. - 'lang': { - type: 'value' - }, - - 'queue-size': { - type: 'value', - validate(size) { - if (parseInt(size) !== parseFloat(size)) return 'an integer'; - if (parseInt(size) < 0) return 'a counting number or zero'; - return true; - } - }, - queue: {alias: 'queue-size'}, - - [parseOptions.handleUnknown]: () => {} - }); + const languageDataFiles = + (await readdir(langPath)) + .filter(name => ['.json', '.yaml'].includes(path.extname(name))) + .map(name => path.join(langPath, name)); - dataPath = miscOptions['data-path'] || process.env.HSMUSIC_DATA; - mediaPath = miscOptions['media-path'] || process.env.HSMUSIC_MEDIA; - langPath = miscOptions['lang-path'] || process.env.HSMUSIC_LANG; // Can 8e left unset! - outputPath = miscOptions['out-path'] || process.env.HSMUSIC_OUT; + let errorLoadingCustomLanguages = false; - const writeOneLanguage = miscOptions['lang']; - - { - let errored = false; - const error = (cond, msg) => { - if (cond) { - console.error(`\x1b[31;1m${msg}\x1b[0m`); - errored = true; - } - }; - error(!dataPath, `Expected --data option or HSMUSIC_DATA to be set`); - error(!mediaPath, `Expected --media option or HSMUSIC_MEDIA to be set`); - error(!outputPath, `Expected --out option or HSMUSIC_OUT to be set`); - if (errored) { - return; - } - } + if (languageReloading) watchCustomLanguages: { + Object.assign(stepStatusSummary.watchLanguageFiles, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - const skipThumbs = miscOptions['skip-thumbs'] ?? false; - const thumbsOnly = miscOptions['thumbs-only'] ?? false; + customLanguageWatchers = + languageDataFiles.map(file => { + const watcher = watchLanguageFile(file); - if (skipThumbs && thumbsOnly) { - logInfo`Well, you've put yourself rather between a roc and a hard place, hmmmm?`; - return; - } + // Bypass node.js special-case handling for uncaught error events + watcher.on('error', () => {}); - if (skipThumbs) { - logInfo`Skipping thumbnail generation.`; - } else { - logInfo`Begin thumbnail generation... -----+`; - const result = await genThumbs(mediaPath, {queueSize, quiet: true}); - logInfo`Done thumbnail generation! --------+`; - if (!result) return; - if (thumbsOnly) return; - } + return watcher; + }); - const defaultStrings = await processLanguageFile(path.join(__dirname, DEFAULT_STRINGS_FILE)); - if (defaultStrings.error) { - logError`Error loading default strings: ${defaultStrings.error}`; - return; - } + const waitingOnWatchers = new Set(customLanguageWatchers); + + const initialResults = + await Promise.allSettled( + customLanguageWatchers + .map(watcher => new Promise((resolve, reject) => { + const onReady = () => { + watcher.removeListener('ready', onReady); + watcher.removeListener('error', onError); + waitingOnWatchers.delete(watcher); + resolve(); + }; + + const onError = error => { + watcher.removeListener('ready', onReady); + watcher.removeListener('error', onError); + reject(error); + }; + + watcher.on('ready', onReady); + watcher.on('error', onError); + }))); + + if (initialResults.some(({status}) => status === 'rejected')) { + logWarn`There were errors loading custom languages from the language path`; + logWarn`provided: ${langPath}`; + + if (noInput) { + internalDefaultLanguageWatcher.close(); + + for (const watcher of Object.values(customLanguageWatchers)) { + watcher.close(); + } + + Object.assign(stepStatusSummary.watchLanguageFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + }); + + errorLoadingCustomLanguages = true; + break watchCustomLanguages; + } + + logWarn`The build should start automatically if you investigate these.`; + logWarn`Or, exit by pressing ^C here (control+C) and run again without`; + logWarn`providing ${'--lang-path'} (or ${'HSMUSIC_LANG'}) to build without custom`; + logWarn`languages.`; + + await new Promise(resolve => { + for (const watcher of waitingOnWatchers) { + watcher.once('ready', () => { + waitingOnWatchers.remove(watcher); + if (empty(waitingOnWatchers)) { + resolve(); + } + }); + } + }); + } - if (langPath) { - const languageDataFiles = await findFiles(langPath, f => path.extname(f) === '.json'); - const results = await progressPromiseAll(`Reading & processing language files.`, languageDataFiles - .map(file => processLanguageFile(file, defaultStrings))); - - let error = false; - for (const strings of results) { - if (strings.error) { - logError`Error loading provided strings: ${strings.error}`; - error = true; - } - } - if (error) return; + languages = + Object.fromEntries( + customLanguageWatchers + .map(({language}) => [language.code, language])); - languages = Object.fromEntries(results.map(strings => [strings.code, strings])); + Object.assign(stepStatusSummary.watchLanguageFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); } else { - languages = {}; - } + languages = {}; - if (!languages[defaultStrings.code]) { - languages[defaultStrings.code] = defaultStrings; - } + const results = + await Promise.allSettled( + languageDataFiles + .map(file => processLanguageFile(file))); - logInfo`Loaded language strings: ${Object.keys(languages).join(', ')}`; + for (const {status, value: language, reason: error} of results) { + if (status === 'rejected') { + errorLoadingCustomLanguages = true; + niceShowAggregate(error); + } else { + languages[language.code] = language; + } + } - if (writeOneLanguage && !(writeOneLanguage in languages)) { - logError`Specified to write only ${writeOneLanguage}, but there is no strings file with this language code!`; - return; - } else if (writeOneLanguage) { - logInfo`Writing only language ${writeOneLanguage} this run.`; - } else { - logInfo`Writing all languages.`; + if (errorLoadingCustomLanguages) { + Object.assign(stepStatusSummary.loadLanguageFiles, { + status: STATUS_FATAL_ERROR, + annotation: `see log for details`, + timeEnd: Date.now(), + }); + } else { + Object.assign(stepStatusSummary.loadLanguageFiles, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } } - WD.wikiInfo = await processWikiInfoFile(path.join(dataPath, WIKI_INFO_FILE)); - if (WD.wikiInfo.error) { - console.log(`\x1b[31;1m${WD.wikiInfo.error}\x1b[0m`); - return; + if (errorLoadingCustomLanguages) { + logError`Failed to load language files. Please investigate these, or don't provide`; + logError`--lang-path (or HSMUSIC_LANG) and build again.`; + return false; } + } else { + languages = {}; + } - // Update languages o8ject with the wiki-specified default language! - // This will make page files for that language 8e gener8ted at the root - // directory, instead of the language-specific su8directory. - if (WD.wikiInfo.defaultLanguage) { - if (Object.keys(languages).includes(WD.wikiInfo.defaultLanguage)) { - languages.default = languages[WD.wikiInfo.defaultLanguage]; - } else { - logError`Wiki info file specified default language is ${WD.wikiInfo.defaultLanguage}, but no such language file exists!`; - if (langPath) { - logError`Check if an appropriate file exists in ${langPath}?`; - } else { - logError`Be sure to specify ${'--lang'} or ${'HSMUSIC_LANG'} with the path to language files.`; - } - return; - } - } else { - languages.default = defaultStrings; - } + Object.assign(stepStatusSummary.initializeDefaultLanguage, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - WD.homepageInfo = await processHomepageInfoFile(path.join(dataPath, HOMEPAGE_INFO_FILE)); + let finalDefaultLanguage; + let finalDefaultLanguageWatcher; + let finalDefaultLanguageAnnotation; - if (WD.homepageInfo.error) { - console.log(`\x1b[31;1m${WD.homepageInfo.error}\x1b[0m`); - return; - } + if (wikiData.wikiInfo.defaultLanguage) { + const customDefaultLanguage = languages[wikiData.wikiInfo.defaultLanguage]; - { - const errors = WD.homepageInfo.rows.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } + if (!customDefaultLanguage) { + logError`Wiki info file specified default language is ${wikiData.wikiInfo.defaultLanguage}, but no such language file exists!`; + if (langPath) { + logError`Check if an appropriate file exists in ${langPath}?`; + } else { + logError`Be sure to specify ${'--lang-path'} or ${'HSMUSIC_LANG'} with the path to language files.`; + } - // 8ut wait, you might say, how do we know which al8um these data files - // correspond to???????? You wouldn't dare suggest we parse the actual - // paths returned 8y this function, which ought to 8e of effectively - // unknown format except for their purpose as reada8le data files!? - // To that, I would say, yeah, you're right. Thanks a 8unch, my projection - // of "you". We're going to read these files later, and contained within - // will 8e the actual directory names that the data correspond to. Yes, - // that's redundant in some ways - we COULD just return the directory name - // in addition to the data path, and duplicating that name within the file - // itself suggests we 8e careful to avoid mismatching it - 8ut doing it - // this way lets the data files themselves 8e more porta8le (meaning we - // could store them all in one folder, if we wanted, and this program would - // still output to the correct al8um directories), and also does make the - // function's signature simpler (an array of strings, rather than some kind - // of structure containing 8oth data file paths and output directories). - // This is o8jectively a good thing, 8ecause it means the function can stay - // truer to its name, and have a narrower purpose: it doesn't need to - // concern itself with where we *output* files, or whatever other reasons - // we might (hypothetically) have for knowing the containing directory. - // And, in the strange case where we DO really need to know that info, we - // callers CAN use path.dirname to find out that data. 8ut we'll 8e - // avoiding that in our code 8ecause, again, we want to avoid assuming the - // format of the returned paths here - they're only meant to 8e used for - // reading as-is. - const albumDataFiles = await findFiles(path.join(dataPath, DATA_ALBUM_DIRECTORY)); - - // Technically, we could do the data file reading and output writing at the - // same time, 8ut that kinda makes the code messy, so I'm not 8othering - // with it. - WD.albumData = await progressPromiseAll(`Reading & processing album files.`, albumDataFiles.map(processAlbumDataFile)); - - { - const errors = WD.albumData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } + Object.assign(stepStatusSummary.initializeDefaultLanguage, { + status: STATUS_FATAL_ERROR, + annotation: `wiki specifies default language whose file is not available`, + timeEnd: Date.now(), + }); - sortByDate(WD.albumData); - - WD.artistData = await processArtistDataFile(path.join(dataPath, ARTIST_DATA_FILE)); - if (WD.artistData.error) { - console.log(`\x1b[31;1m${WD.artistData.error}\x1b[0m`); - return; + return false; } - { - const errors = WD.artistData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } + logInfo`Applying new default strings from custom ${customDefaultLanguage.code} language file.`; - WD.artistAliasData = WD.artistData.filter(x => x.alias); - WD.artistData = WD.artistData.filter(x => !x.alias); + finalDefaultLanguage = customDefaultLanguage; + finalDefaultLanguageAnnotation = `using wiki-specified custom default language`; - WD.trackData = getAllTracks(WD.albumData); + if (languageReloading) { + finalDefaultLanguageWatcher = + customLanguageWatchers + .find(({language}) => language === customDefaultLanguage); + } + } else if (languages[internalDefaultLanguage.code]) { + const customDefaultLanguage = languages[internalDefaultLanguage.code]; - if (WD.wikiInfo.features.flashesAndGames) { - WD.flashData = await processFlashDataFile(path.join(dataPath, FLASH_DATA_FILE)); - if (WD.flashData.error) { - console.log(`\x1b[31;1m${WD.flashData.error}\x1b[0m`); - return; - } + finalDefaultLanguage = customDefaultLanguage; + finalDefaultLanguageAnnotation = `using inferred custom default language`; - const errors = WD.flashData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } + if (languageReloading) { + finalDefaultLanguageWatcher = + customLanguageWatchers + .find(({language}) => language === customDefaultLanguage); } + } else { + languages[internalDefaultLanguage.code] = internalDefaultLanguage; - WD.flashActData = WD.flashData?.filter(x => x.act8r8k); - WD.flashData = WD.flashData?.filter(x => !x.act8r8k); + finalDefaultLanguage = internalDefaultLanguage; + finalDefaultLanguageAnnotation = `no custom default language specified`; - WD.tagData = await processTagDataFile(path.join(dataPath, TAG_DATA_FILE)); - if (WD.tagData.error) { - console.log(`\x1b[31;1m${WD.tagData.error}\x1b[0m`); - return; + if (languageReloading) { + finalDefaultLanguageWatcher = internalDefaultLanguageWatcher; } + } - { - const errors = WD.tagData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } + const closeLanguageWatchers = () => { + if (languageReloading) { + for (const watcher of [ + internalDefaultLanguageWatcher, + ...customLanguageWatchers, + ]) { + watcher.close(); + } } + }; - WD.tagData.sort(sortByName); + const inheritStringsFromInternalLanguage = () => { + // The custom default language, if set, will be the new one providing fallback + // strings for other languages. But on its own, it still might not be a complete + // list of strings - so it falls back to the internal default language, which + // won't otherwise be presented in the build. + if (finalDefaultLanguage === internalDefaultLanguage) return; + const {strings: inheritedStrings} = internalDefaultLanguage; + Object.assign(finalDefaultLanguage, {inheritedStrings}); + }; - WD.groupData = await processGroupDataFile(path.join(dataPath, GROUP_DATA_FILE)); - if (WD.groupData.error) { - console.log(`\x1b[31;1m${WD.groupData.error}\x1b[0m`); - return; + const inheritStringsFromDefaultLanguage = () => { + const {strings: inheritedStrings} = finalDefaultLanguage; + for (const language of Object.values(languages)) { + if (language === finalDefaultLanguage) continue; + Object.assign(language, {inheritedStrings}); } + }; - { - const errors = WD.groupData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } + if (finalDefaultLanguage !== internalDefaultLanguage) { + inheritStringsFromInternalLanguage(); + } - WD.groupCategoryData = WD.groupData.filter(x => x.isCategory); - WD.groupData = WD.groupData.filter(x => x.isGroup); + inheritStringsFromDefaultLanguage(); - WD.staticPageData = await processStaticPageDataFile(path.join(dataPath, STATIC_PAGE_DATA_FILE)); - if (WD.staticPageData.error) { - console.log(`\x1b[31;1m${WD.staticPageData.error}\x1b[0m`); - return; + if (languageReloading) { + if (finalDefaultLanguage !== internalDefaultLanguage) { + internalDefaultLanguageWatcher.on('update', () => { + inheritStringsFromInternalLanguage(); + inheritStringsFromDefaultLanguage(); + }); } - { - const errors = WD.staticPageData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } - } + finalDefaultLanguageWatcher.on('update', () => { + inheritStringsFromDefaultLanguage(); + }); + } - if (WD.wikiInfo.features.news) { - WD.newsData = await processNewsDataFile(path.join(dataPath, NEWS_DATA_FILE)); - if (WD.newsData.error) { - console.log(`\x1b[31;1m${WD.newsData.error}\x1b[0m`); - return; - } + logInfo`Loaded language strings: ${Object.keys(languages).join(', ')}`; - const errors = WD.newsData.filter(obj => obj.error); - if (errors.length) { - for (const error of errors) { - console.log(`\x1b[31;1m${error.error}\x1b[0m`); - } - return; - } + Object.assign(stepStatusSummary.initializeDefaultLanguage, { + status: STATUS_DONE_CLEAN, + annotation: finalDefaultLanguageAnnotation, + timeEnd: Date.now(), + }); - sortByDate(WD.newsData); - WD.newsData.reverse(); - } - - { - const tagNames = new Set([...WD.trackData, ...WD.albumData].flatMap(thing => thing.artTags)); + const urls = generateURLs(urlSpec); - for (let { name, isCW } of WD.tagData) { - if (isCW) { - name = 'cw: ' + name; - } - tagNames.delete(name); - } + let missingImagePaths; - if (tagNames.size) { - for (const name of Array.from(tagNames).sort()) { - console.log(`\x1b[33;1m- Missing tag: "${name}"\x1b[0m`); - } - return; - } - } + if (stepStatusSummary.verifyImagePaths.status === STATUS_NOT_APPLICABLE) { + missingImagePaths = []; + } else if (stepStatusSummary.verifyImagePaths.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.verifyImagePaths, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - WD.justEverythingMan = sortByDate([...WD.albumData, ...WD.trackData, ...(WD.flashData || [])]); - WD.justEverythingSortedByArtDateMan = sortByArtDate(WD.justEverythingMan.slice()); - // console.log(JSON.stringify(justEverythingSortedByArtDateMan.map(toAnythingMan), null, 2)); - - const artistNames = Array.from(new Set([ - ...WD.artistData.filter(artist => !artist.alias).map(artist => artist.name), - ...[ - ...WD.albumData.flatMap(album => [ - ...album.artists || [], - ...album.coverArtists || [], - ...album.wallpaperArtists || [], - ...album.tracks.flatMap(track => [ - ...track.artists, - ...track.coverArtists || [], - ...track.contributors || [] - ]) - ]), - ...(WD.flashData?.flatMap(flash => [ - ...flash.contributors || [] - ]) || []) - ].map(contribution => contribution.who) - ])); - - artistNames.sort((a, b) => a.toLowerCase() < b.toLowerCase() ? -1 : a.toLowerCase() > b.toLowerCase() ? 1 : 0); - - { - let buffer = []; - const clearBuffer = function() { - if (buffer.length) { - for (const entry of buffer.slice(0, -1)) { - console.log(`\x1b[2m... ${entry.name} ...\x1b[0m`); - } - const lastEntry = buffer[buffer.length - 1]; - console.log(`\x1b[2m... \x1b[0m${lastEntry.name}\x1b[0;2m ...\x1b[0m`); - buffer = []; - } - }; - const showWhere = (name, color) => { - const where = WD.justEverythingMan.filter(thing => [ - ...thing.coverArtists || [], - ...thing.contributors || [], - ...thing.artists || [] - ].some(({ who }) => who === name)); - for (const thing of where) { - console.log(`\x1b[${color}m- ` + (thing.album ? `(\x1b[1m${thing.album.name}\x1b[0;${color}m)` : '') + ` \x1b[1m${thing.name}\x1b[0m`); - } - }; - let CR4SH = false; - for (let name of artistNames) { - const entry = [...WD.artistData, ...WD.artistAliasData].find(entry => entry.name === name || entry.name.toLowerCase() === name.toLowerCase()); - if (!entry) { - clearBuffer(); - console.log(`\x1b[31mMissing entry for artist "\x1b[1m${name}\x1b[0;31m"\x1b[0m`); - showWhere(name, 31); - CR4SH = true; - } else if (entry.alias) { - console.log(`\x1b[33mArtist "\x1b[1m${name}\x1b[0;33m" should be named "\x1b[1m${entry.alias}\x1b[0;33m"\x1b[0m`); - showWhere(name, 33); - CR4SH = true; - } else if (entry.name !== name) { - console.log(`\x1b[33mArtist "\x1b[1m${name}\x1b[0;33m" should be named "\x1b[1m${entry.name}\x1b[0;33m"\x1b[0m`); - showWhere(name, 33); - CR4SH = true; - } else { - buffer.push(entry); - if (buffer.length > 3) { - buffer.shift(); - } - } - } - if (CR4SH) { - return; - } - } + const results = + await verifyImagePaths(mediaPath, {urls, wikiData}); + + missingImagePaths = results.missing; + const misplacedImagePaths = results.misplaced; + + if (empty(missingImagePaths) && empty(misplacedImagePaths)) { + Object.assign(stepStatusSummary.verifyImagePaths, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } else if (empty(missingImagePaths)) { + Object.assign(stepStatusSummary.verifyImagePaths, { + status: STATUS_HAS_WARNINGS, + annotation: `misplaced images detected`, + timeEnd: Date.now(), + }); + } else if (empty(misplacedImagePaths)) { + Object.assign(stepStatusSummary.verifyImagePaths, { + status: STATUS_HAS_WARNINGS, + annotation: `missing images detected`, + timeEnd: Date.now(), + }); + } else { + Object.assign(stepStatusSummary.verifyImagePaths, { + status: STATUS_HAS_WARNINGS, + annotation: `missing and misplaced images detected`, + timeEnd: Date.now(), + }); + } + } + + let getSizeOfAdditionalFile; + let getSizeOfImagePath; + + if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_APPLICABLE) { + getSizeOfAdditionalFile = () => null; + getSizeOfImagePath = () => null; + } else if (stepStatusSummary.preloadFileSizes.status === STATUS_NOT_STARTED) { + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); - { - const directories = []; - for (const { directory, name } of WD.albumData) { - if (directories.includes(directory)) { - console.log(`\x1b[31;1mDuplicate album directory "${directory}" (${name})\x1b[0m`); - return; - } - directories.push(directory); - } - } + const fileSizePreloader = new FileSizePreloader(); + + // File sizes of additional files need to be precalculated before we can + // actually reference 'em in site building, so get those loading right + // away. We actually need to keep track of two things here - the on-device + // file paths we're actually reading, and the corresponding on-site media + // paths that will be exposed in site build code. We'll build a mapping + // function between them so that when site code requests a site path, + // it'll get the size of the file at the corresponding device path. + const additionalFilePaths = [ + ...wikiData.albumData.flatMap((album) => + [ + ...(album.additionalFiles ?? []), + ...album.tracks.flatMap((track) => [ + ...(track.additionalFiles ?? []), + ...(track.sheetMusicFiles ?? []), + ...(track.midiProjectFiles ?? []), + ]), + ] + .flatMap((fileGroup) => fileGroup.files ?? []) + .map((file) => ({ + device: path.join( + mediaPath, + urls + .from('media.root') + .toDevice('media.albumAdditionalFile', album.directory, file) + ), + media: urls + .from('media.root') + .to('media.albumAdditionalFile', album.directory, file), + })) + ), + ]; + + // Same dealio for images. Since just about any image can be embedded and + // we can't super easily know which ones are referenced at runtime, just + // cheat and get file sizes for all images under media. (This includes + // additional files which are images.) + const imageFilePaths = + await traverse(mediaPath, { + pathStyle: 'device', + filterDir: dir => dir !== '.git', + filterFile: file => + ['.png', '.gif', '.jpg'].includes(path.extname(file)) && + !isThumb(file), + }).then(files => files + .map(file => ({ + device: file, + media: + urls + .from('media.root') + .to('media.path', path.relative(mediaPath, file).split(path.sep).join('/')), + }))); + + const getSizeOfMediaFileHelper = paths => (mediaPath) => { + const pair = paths.find(({media}) => media === mediaPath); + if (!pair) return null; + return fileSizePreloader.getSizeOfPath(pair.device); + }; - { - const directories = []; - const where = {}; - for (const { directory, album } of WD.trackData) { - if (directories.includes(directory)) { - console.log(`\x1b[31;1mDuplicate track directory "${directory}"\x1b[0m`); - console.log(`Shows up in:`); - console.log(`- ${album.name}`); - console.log(`- ${where[directory].name}`); - return; - } - directories.push(directory); - where[directory] = album; - } - } + getSizeOfAdditionalFile = getSizeOfMediaFileHelper(additionalFilePaths); + getSizeOfImagePath = getSizeOfMediaFileHelper(imageFilePaths); - { - const artists = []; - const artistsLC = []; - for (const name of artistNames) { - if (!artists.includes(name) && artistsLC.includes(name.toLowerCase())) { - const other = artists.find(oth => oth.toLowerCase() === name.toLowerCase()); - console.log(`\x1b[31;1mMiscapitalized artist name: ${name}, ${other}\x1b[0m`); - return; - } - artists.push(name); - artistsLC.push(name.toLowerCase()); - } - } + logInfo`Preloading filesizes for ${additionalFilePaths.length} additional files...`; - { - for (const { references, name, album } of WD.trackData) { - for (const ref of references) { - if (!find.track(ref, {wikiData})) { - logWarn`Track not found "${ref}" in ${name} (${album.name})`; - } - } - } - } + fileSizePreloader.loadPaths(...additionalFilePaths.map((path) => path.device)); + await fileSizePreloader.waitUntilDoneLoading(); - WD.contributionData = Array.from(new Set([ - ...WD.trackData.flatMap(track => [...track.artists || [], ...track.contributors || [], ...track.coverArtists || []]), - ...WD.albumData.flatMap(album => [...album.artists || [], ...album.coverArtists || [], ...album.wallpaperArtists || [], ...album.bannerArtists || []]), - ...(WD.flashData?.flatMap(flash => [...flash.contributors || []]) || []) - ])); - - // Now that we have all the data, resolve references all 8efore actually - // gener8ting any of the pages, 8ecause page gener8tion is going to involve - // accessing these references a lot, and there's no reason to resolve them - // more than once. (We 8uild a few additional links that can't 8e cre8ted - // at initial data processing time here too.) - - const filterNullArray = (parent, key) => { - for (const obj of parent) { - const array = obj[key]; - for (let i = 0; i < array.length; i++) { - if (!array[i]) { - const prev = array[i - 1] && array[i - 1].name; - const next = array[i + 1] && array[i + 1].name; - logWarn`Unexpected null in ${obj.name} (${obj.what}) (array key ${key} - prev: ${prev}, next: ${next})`; - } - } - array.splice(0, array.length, ...array.filter(Boolean)); - } - }; + logInfo`Preloading filesizes for ${imageFilePaths.length} full-resolution images...`; - const filterNullValue = (parent, key) => { - parent.splice(0, parent.length, ...parent.filter(obj => { - if (!obj[key]) { - logWarn`Unexpected null in ${obj.name} (value key ${key})`; - return false; - } - return true; - })); - }; + fileSizePreloader.loadPaths(...imageFilePaths.map((path) => path.device)); + await fileSizePreloader.waitUntilDoneLoading(); - WD.trackData.forEach(track => mapInPlace(track.references, r => find.track(r, {wikiData}))); - WD.trackData.forEach(track => track.aka = find.track(track.aka, {wikiData})); - WD.trackData.forEach(track => mapInPlace(track.artTags, t => find.tag(t, {wikiData}))); - WD.albumData.forEach(album => mapInPlace(album.groups, g => find.group(g, {wikiData}))); - WD.albumData.forEach(album => mapInPlace(album.artTags, t => find.tag(t, {wikiData}))); - WD.artistAliasData.forEach(artist => artist.alias = find.artist(artist.alias, {wikiData})); - WD.contributionData.forEach(contrib => contrib.who = find.artist(contrib.who, {wikiData})); - - filterNullArray(WD.trackData, 'references'); - filterNullArray(WD.trackData, 'artTags'); - filterNullArray(WD.albumData, 'groups'); - filterNullArray(WD.albumData, 'artTags'); - filterNullValue(WD.artistAliasData, 'alias'); - filterNullValue(WD.contributionData, 'who'); - - WD.trackData.forEach(track1 => track1.referencedBy = WD.trackData.filter(track2 => track2.references.includes(track1))); - WD.groupData.forEach(group => group.albums = WD.albumData.filter(album => album.groups.includes(group))); - WD.tagData.forEach(tag => tag.things = sortByArtDate([...WD.albumData, ...WD.trackData]).filter(thing => thing.artTags.includes(tag))); - - WD.groupData.forEach(group => group.category = WD.groupCategoryData.find(x => x.name === group.category)); - WD.groupCategoryData.forEach(category => category.groups = WD.groupData.filter(x => x.category === category)); - - WD.trackData.forEach(track => track.otherReleases = [ - track.aka, - ...WD.trackData.filter(({ aka }) => aka === track || (track.aka && aka === track.aka)), - ].filter(x => x && x !== track)); - - if (WD.wikiInfo.features.flashesAndGames) { - WD.flashData.forEach(flash => mapInPlace(flash.tracks, t => find.track(t, {wikiData}))); - WD.flashData.forEach(flash => flash.act = WD.flashActData.find(act => act.name === flash.act)); - WD.flashActData.forEach(act => act.flashes = WD.flashData.filter(flash => flash.act === act)); - - filterNullArray(WD.flashData, 'tracks'); - - WD.trackData.forEach(track => track.flashes = WD.flashData.filter(flash => flash.tracks.includes(track))); - } + if (fileSizePreloader.hasErrored) { + logWarn`Some media files couldn't be read for preloading filesizes.`; + logWarn`This means the wiki won't display file sizes for these files.`; + logWarn`Investigate missing or unreadable files to get that fixed!`; - WD.artistData.forEach(artist => { - const filterProp = (array, prop) => array.filter(thing => thing[prop]?.some(({ who }) => who === artist)); - const filterCommentary = array => array.filter(thing => thing.commentary && thing.commentary.replace(/<\/?b>/g, '').includes('<i>' + artist.name + ':</i>')); - artist.tracks = { - asArtist: filterProp(WD.trackData, 'artists'), - asCommentator: filterCommentary(WD.trackData), - asContributor: filterProp(WD.trackData, 'contributors'), - asCoverArtist: filterProp(WD.trackData, 'coverArtists'), - asAny: WD.trackData.filter(track => ( - [...track.artists, ...track.contributors, ...track.coverArtists || []].some(({ who }) => who === artist) - )) - }; - artist.albums = { - asArtist: filterProp(WD.albumData, 'artists'), - asCommentator: filterCommentary(WD.albumData), - asCoverArtist: filterProp(WD.albumData, 'coverArtists'), - asWallpaperArtist: filterProp(WD.albumData, 'wallpaperArtists'), - asBannerArtist: filterProp(WD.albumData, 'bannerArtists') - }; - if (WD.wikiInfo.features.flashesAndGames) { - artist.flashes = { - asContributor: filterProp(WD.flashData, 'contributors') - }; - } + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_HAS_WARNINGS, + annotation: `see log for details`, + timeEnd: Date.now(), + }); + } else { + logInfo`Done preloading filesizes without any errors - nice!`; + + Object.assign(stepStatusSummary.preloadFileSizes, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); + } + } + + if (stepStatusSummary.performBuild.status === STATUS_NOT_APPLICABLE) { + return true; + } + + const developersComment = + `<!--\n` + [ + wikiData.wikiInfo.canonicalBase + ? `hsmusic.wiki - ${wikiData.wikiInfo.name}, ${wikiData.wikiInfo.canonicalBase}` + : `hsmusic.wiki - ${wikiData.wikiInfo.name}`, + 'Code copyright 2019-2023 Quasar Nebula et al (MIT License)', + ...wikiData.wikiInfo.canonicalBase === 'https://hsmusic.wiki/' ? [ + 'Data avidly compiled and localization brought to you', + 'by our awesome team and community of wiki contributors', + '***', + 'Want to contribute? Join our Discord or leave feedback!', + '- https://hsmusic.wiki/discord/', + '- https://hsmusic.wiki/feedback/', + '- https://github.com/hsmusic/', + ] : [ + 'https://github.com/hsmusic/', + ], + '***', + BUILD_TIME && + `Site built: ${BUILD_TIME.toLocaleString('en-US', { + dateStyle: 'long', + timeStyle: 'long', + })}`, + COMMIT && + `Latest code commit: ${COMMIT}`, + ] + .filter(Boolean) + .map(line => ` ` + line) + .join('\n') + `\n-->`; + + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_STARTED_NOT_DONE, + timeStart: Date.now(), + }); + + let buildModeResult; + + try { + buildModeResult = await selectedBuildMode.go({ + cliOptions, + dataPath, + mediaPath, + mediaCachePath, + queueSize, + srcRootPath: __dirname, + + defaultLanguage: finalDefaultLanguage, + languages, + missingImagePaths, + thumbsCache, + urls, + urlSpec, + wikiData, + + cachebust: '?' + CACHEBUST, + closeLanguageWatchers, + developersComment, + getSizeOfAdditionalFile, + getSizeOfImagePath, + niceShowAggregate, }); + } catch (error) { + console.error(error); - WD.officialAlbumData = WD.albumData.filter(album => album.groups.some(group => group.directory === OFFICIAL_GROUP_DIRECTORY)); - WD.fandomAlbumData = WD.albumData.filter(album => album.groups.every(group => group.directory !== OFFICIAL_GROUP_DIRECTORY)); + logError`There was a JavaScript error performing the build.`; + fileIssue(); - // Makes writing a little nicer on CPU theoretically, 8ut also costs in - // performance right now 'cuz it'll w8 for file writes to 8e completed - // 8efore moving on to more data processing. So, defaults to zero, which - // disa8les the queue feature altogether. - queueSize = +(miscOptions['queue-size'] ?? 0); + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_FATAL_ERROR, + message: `javascript error - view log for details`, + timeEnd: Date.now(), + }); - const buildDictionary = pageSpecs; + return false; + } - // NOT for ena8ling or disa8ling specific features of the site! - // This is only in charge of what general groups of files to 8uild. - // They're here to make development quicker when you're only working - // on some particular area(s) of the site rather than making changes - // across all of them. - const writeFlags = await parseOptions(process.argv.slice(2), { - all: {type: 'flag'}, // Defaults to true if none 8elow specified. + if (buildModeResult !== true) { + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_HAS_WARNINGS, + annotation: `may not have completed - view log for details`, + timeEnd: Date.now(), + }); - // Kinda a hack t8h! - ...Object.fromEntries(Object.keys(buildDictionary) - .map(key => [key, {type: 'flag'}])), + return false; + } - [parseOptions.handleUnknown]: () => {} - }); + Object.assign(stepStatusSummary.performBuild, { + status: STATUS_DONE_CLEAN, + timeEnd: Date.now(), + }); - const writeAll = !Object.keys(writeFlags).length || writeFlags.all; + return true; +} - logInfo`Writing site pages: ${writeAll ? 'all' : Object.keys(writeFlags).join(', ')}`; +// TODO: isMain detection isn't consistent across platforms here +/* eslint-disable-next-line no-constant-condition */ +if (true || isMain(import.meta.url) || path.basename(process.argv[1]) === 'hsmusic') { + (async () => { + let result; - await writeSymlinks(); - await writeSharedFilesAndPages({strings: defaultStrings, wikiData}); + const totalTimeStart = Date.now(); - const buildSteps = (writeAll - ? Object.entries(buildDictionary) - : (Object.entries(buildDictionary) - .filter(([ flag ]) => writeFlags[flag]))); + try { + result = await main(); + } catch (error) { + if (error instanceof AggregateError) { + showAggregate(error); + } else if (error.cause) { + console.error(error); + showAggregate(error); + } else { + console.error(error); + } + } - let writes; - { - let error = false; + const totalTimeEnd = Date.now(); - const buildStepsWithTargets = buildSteps.map(([ flag, pageSpec ]) => { - // Condition not met: skip this build step altogether. - if (pageSpec.condition && !pageSpec.condition({wikiData})) { - return null; - } + const formatDuration = timeDelta => { + const seconds = timeDelta / 1000; - // May still call writeTargetless if present. - if (!pageSpec.targets) { - return {flag, pageSpec, targets: []}; - } + if (seconds > 90) { + const modSeconds = Math.floor(seconds % 60); + const minutes = Math.floor(seconds - seconds % 60) / 60; + return `${minutes}m${modSeconds}s`; + } - if (!pageSpec.write) { - logError`${flag + '.targets'} is specified, but ${flag + '.write'} is missing!`; - error = true; - return null; - } + if (seconds < 0.1) { + return 'instant'; + } - const targets = pageSpec.targets({wikiData}); - return {flag, pageSpec, targets}; - }).filter(Boolean); + const precision = (seconds > 1 ? 3 : 2); + return `${seconds.toPrecision(precision)}s`; + }; - if (error) { - return; - } + if (showStepStatusSummary) { + const totalDuration = formatDuration(totalTimeEnd - totalTimeStart); - const validateWrites = (writes, fnName) => { - // Do a quick valid8tion! If one of the writeThingPages functions go - // wrong, this will stall out early and tell us which did. - - if (!Array.isArray(writes)) { - logError`${fnName} didn't return an array!`; - error = true; - return false; - } - - if (!( - writes.every(obj => typeof obj === 'object') && - writes.every(obj => { - const result = validateWriteObject(obj); - if (result.error) { - logError`Validating write object failed: ${result.error}`; - return false; - } else { - return true; - } - }) - )) { - logError`${fnName} returned invalid entries!`; - error = true; - return false; - } - - return true; - }; + console.error(colors.bright(`Step summary:`)); + + const longestNameLength = + Math.max(... + Object.values(stepStatusSummary) + .map(({name}) => name.length)); - writes = buildStepsWithTargets.flatMap(({ flag, pageSpec, targets }) => { - const writes = targets.flatMap(target => - pageSpec.write(target, {wikiData}).slice() || []); + const stepsNotClean = + Object.values(stepStatusSummary) + .map(({status}) => + status === STATUS_HAS_WARNINGS || + status === STATUS_FATAL_ERROR || + status === STATUS_STARTED_NOT_DONE); - if (!validateWrites(writes, flag + '.write')) { - return []; - } + const anyStepsNotClean = + stepsNotClean.includes(true); - if (pageSpec.writeTargetless) { - const writes2 = pageSpec.writeTargetless({wikiData}); + const stepDetails = Object.values(stepStatusSummary); - if (!validateWrites(writes2, flag + '.writeTargetless')) { - return []; - } + const stepDurations = + stepDetails.map(({status, timeStart, timeEnd}) => { + if ( + status === STATUS_NOT_APPLICABLE || + status === STATUS_NOT_STARTED || + status === STATUS_STARTED_NOT_DONE + ) { + return '-'; + } - writes.push(...writes2); - } + if (typeof timeStart !== 'number' || typeof timeEnd !== 'number') { + return 'unknown'; + } - return writes; + return formatDuration(timeEnd - timeStart); }); - if (error) { - return; - } - } + const longestDurationLength = + Math.max(...stepDurations.map(duration => duration.length)); - const pageWrites = writes.filter(({ type }) => type === 'page'); - const dataWrites = writes.filter(({ type }) => type === 'data'); - const redirectWrites = writes.filter(({ type }) => type === 'redirect'); + for (let index = 0; index < stepDetails.length; index++) { + const {name, status, annotation} = stepDetails[index]; + const duration = stepDurations[index]; - if (writes.length) { - logInfo`Total of ${writes.length} writes returned. (${pageWrites.length} page, ${dataWrites.length} data, ${redirectWrites.length} redirect)`; - } else { - logWarn`No writes returned at all, so exiting early. This is probably a bug!`; - return; - } + let message = + (stepsNotClean[index] + ? `!! ` + : ` - `); - await progressPromiseAll(`Writing data files shared across languages.`, queue( - dataWrites.map(({path, data}) => () => { - const bound = {}; + message += `(${duration})`.padStart(longestDurationLength + 2, ' '); + message += ` `; + message += `${name}: `.padEnd(longestNameLength + 4, '.'); + message += ` `; + message += status; - bound.serializeLink = bindOpts(serializeLink, {}); + if (annotation) { + message += ` (${annotation})`; + } - bound.serializeContribs = bindOpts(serializeContribs, {}); + switch (status) { + case STATUS_DONE_CLEAN: + console.error(colors.green(message)); + break; - bound.serializeImagePaths = bindOpts(serializeImagePaths, { - thumb - }); + case STATUS_NOT_STARTED: + case STATUS_NOT_APPLICABLE: + console.error(colors.dim(message)); + break; - bound.serializeCover = bindOpts(serializeCover, { - [bindOpts.bindIndex]: 2, - serializeImagePaths: bound.serializeImagePaths, - urls - }); + case STATUS_HAS_WARNINGS: + case STATUS_STARTED_NOT_DONE: + console.error(colors.yellow(message)); + break; - bound.serializeGroupsForAlbum = bindOpts(serializeGroupsForAlbum, { - serializeLink - }); + case STATUS_FATAL_ERROR: + console.error(colors.red(message)); + break; - bound.serializeGroupsForTrack = bindOpts(serializeGroupsForTrack, { - serializeLink - }); + default: + console.error(message); + break; + } + } - // TODO: This only supports one <>-style argument. - return writeData(path[0], path[1], data({ - ...bound - })); - }), - queueSize - )); - - const perLanguageFn = async ({strings, ...opts}, i, entries) => { - console.log(`\x1b[34;1m${ - (`[${i + 1}/${entries.length}] ${strings.code} (-> /${opts.baseDirectory}) ` - .padEnd(60, '-')) - }\x1b[0m`); - - await progressPromiseAll(`Writing ${strings.code}`, queue([ - ...pageWrites.map(({type, ...props}) => () => { - const { path, page } = props; - const { baseDirectory } = opts; - - // TODO: This only supports one <>-style argument. - const pageSubKey = path[0]; - const directory = path[1]; - - const paths = writePage.paths(baseDirectory, 'localized.' + pageSubKey, directory); - const to = writePage.to({baseDirectory, pageSubKey, paths}); - - // TODO: Is there some nicer way to define these, - // may8e without totally re-8inding everything for - // each page? - const bound = {}; - - bound.link = withEntries(unbound_link, entries => entries - .map(([ key, fn ]) => [key, bindOpts(fn, {to})])); - - bound.linkAnythingMan = bindOpts(linkAnythingMan, { - link: bound.link, - wikiData - }); - - bound.parseAttributes = bindOpts(parseAttributes, { - to - }); - - bound.transformInline = bindOpts(transformInline, { - link: bound.link, - replacerSpec, - strings, - to, - wikiData - }); - - bound.transformMultiline = bindOpts(transformMultiline, { - transformInline: bound.transformInline, - parseAttributes: bound.parseAttributes - }); - - bound.transformLyrics = bindOpts(transformLyrics, { - transformInline: bound.transformInline, - transformMultiline: bound.transformMultiline - }); - - bound.iconifyURL = bindOpts(iconifyURL, { - strings, - to - }); - - bound.fancifyURL = bindOpts(fancifyURL, { - strings - }); - - bound.fancifyFlashURL = bindOpts(fancifyFlashURL, { - [bindOpts.bindIndex]: 2, - strings - }); - - bound.getLinkThemeString = getLinkThemeString; - - bound.getThemeString = getThemeString; - - bound.getArtistString = bindOpts(getArtistString, { - iconifyURL: bound.iconifyURL, - link: bound.link, - strings - }); - - bound.getAlbumCover = bindOpts(getAlbumCover, { - to - }); - - bound.getTrackCover = bindOpts(getTrackCover, { - to - }); - - bound.getFlashCover = bindOpts(getFlashCover, { - to - }); - - bound.generateChronologyLinks = bindOpts(generateChronologyLinks, { - link: bound.link, - linkAnythingMan: bound.linkAnythingMan, - strings, - wikiData - }); - - bound.generateCoverLink = bindOpts(generateCoverLink, { - [bindOpts.bindIndex]: 0, - img, - link: bound.link, - strings, - to, - wikiData - }); - - bound.generateInfoGalleryLinks = bindOpts(generateInfoGalleryLinks, { - [bindOpts.bindIndex]: 2, - link: bound.link, - strings - }); - - bound.generatePreviousNextLinks = bindOpts(generatePreviousNextLinks, { - link: bound.link, - strings - }); - - bound.getGridHTML = bindOpts(getGridHTML, { - [bindOpts.bindIndex]: 0, - getLinkThemeString, - img, - strings - }); - - bound.getAlbumGridHTML = bindOpts(getAlbumGridHTML, { - [bindOpts.bindIndex]: 0, - getAlbumCover: bound.getAlbumCover, - getGridHTML: bound.getGridHTML, - strings, - to - }); - - bound.getFlashGridHTML = bindOpts(getFlashGridHTML, { - [bindOpts.bindIndex]: 0, - getFlashCover: bound.getFlashCover, - getGridHTML: bound.getGridHTML, - to - }); - - bound.getRevealStringFromTags = bindOpts(getRevealStringFromTags, { - strings - }); - - bound.getRevealStringFromWarnings = bindOpts(getRevealStringFromWarnings, { - strings - }); - - bound.getAlbumStylesheet = bindOpts(getAlbumStylesheet, { - to - }); - - const pageFn = () => page({ - ...bound, - strings, - to - }); - - const content = writePage.html(pageFn, { - paths, - strings, - to, - transformMultiline: bound.transformMultiline, - wikiData - }); - - return writePage.write(content, {paths}); - }), - ...redirectWrites.map(({fromPath, toPath, title: titleFn}) => () => { - const { baseDirectory } = opts; - - const title = titleFn({ - strings - }); - - // TODO: This only supports one <>-style argument. - const fromPaths = writePage.paths(baseDirectory, 'localized.' + fromPath[0], fromPath[1]); - const to = writePage.to({baseDirectory, pageSubKey: fromPath[0], paths: fromPaths}); - - const target = to('localized.' + toPath[0], ...toPath.slice(1)); - const content = generateRedirectPage(title, target, {strings}); - return writePage.write(content, {paths: fromPaths}); - }) - ], queueSize)); - }; + console.error(colors.bright(`Done in ${totalDuration}.`)); - await wrapLanguages(perLanguageFn, { - writeOneLanguage, - wikiData - }); + if (result === true) { + if (anyStepsNotClean) { + console.error(colors.bright(`Final output is true, but some steps aren't clean.`)); + process.exit(1); + return; + } else { + console.error(colors.bright(`Final output is true and all steps are clean.`)); + } + } else if (result === false) { + console.error(colors.bright(`Final output is false.`)); + } else { + console.error(colors.bright(`Final output is not true (${result}).`)); + } + } + + if (result !== true) { + process.exit(1); + return; + } decorateTime.displayTime(); + CacheableObject.showInvalidAccesses(); - // The single most important step. - logInfo`Written!`; + process.exit(0); + })(); } - -main().catch(error => console.error(error)); |