From d85e07d6fe6a0435fa1b911443d88c36a1b03901 Mon Sep 17 00:00:00 2001 From: ionous Date: Sat, 31 Jan 2026 11:47:50 -0800 Subject: [PATCH 1/7] tweaks to existing tests ex. use testData.expectError where possible --- app/test/manage_test.js | 10 ++-------- app/test/ride_count_test.js | 13 +++++++++---- app/test/search_test.js | 8 ++------ 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/app/test/manage_test.js b/app/test/manage_test.js index 6d1b8865..dfeb72ce 100644 --- a/app/test/manage_test.js +++ b/app/test/manage_test.js @@ -94,10 +94,7 @@ describe("managing events", () => { return request(app) .post(manage_api) .send(post) - .expect(400) - .then(res => { - assert.ok(res.body.error.fields[key]); - }); + .then(res => testData.expectError(res, key)); }) } return seq; @@ -126,10 +123,7 @@ describe("managing events", () => { return request(app) .post(manage_api) .send(post) - .expect(400) - .then(res => { - assert.ok(res.body.error.fields[key]); - }); + .then(res => testData.expectError(res, key)); }) } return seq; diff --git a/app/test/ride_count_test.js b/app/test/ride_count_test.js index 41b66df4..bbc1dc12 100644 --- a/app/test/ride_count_test.js +++ b/app/test/ride_count_test.js @@ -1,5 +1,6 @@ const app = require("../appEndpoints"); const testdb = require("./testdb"); +const testData = require("./testData"); // const { describe, it, before, after } = require("node:test"); const assert = require("node:assert/strict"); @@ -14,6 +15,12 @@ describe("ride count testing", () => { after(() => { return testdb.destroy(); }); + // FIX! it would be good to support an unspecified range + it("handles an unspecified range", () => { + return request(app) + .get('/api/ride_count.php') + .expect(testData.expectError); + }); it("handles an all encompassing range", () => { return request(app) .get('/api/ride_count.php') @@ -42,14 +49,12 @@ describe("ride count testing", () => { it("errors on a missing time", () => { return request(app) .get('/api/ride_count.php') - .expect(400) - .expect('Content-Type', /json/); + .expect(testData.expectError); }); it("errors on an invalid time", () => { return request(app) .get('/api/ride_count.php') .query({s: "yesterday", e: "tomorrow"}) - .expect(400) - .expect('Content-Type', /json/); + .expect(testData.expectError); }); }); diff --git a/app/test/search_test.js b/app/test/search_test.js index c78fcdfc..ce3a8cb4 100644 --- a/app/test/search_test.js +++ b/app/test/search_test.js @@ -1,5 +1,6 @@ const app = require("../appEndpoints"); const testdb = require("./testdb"); +const testData = require("./testData"); const { EventSearch } = require("../models/calConst"); // const { describe, it, before, after } = require("node:test"); @@ -19,12 +20,7 @@ describe("searching for events", () => { it("errors on an empty search term", () => { return request(app) .get('/api/search.php') - // .query({q: "events"}) - .expect(400) - .expect('Content-Type', /json/) - .then(res => { - assert.ok(res.body?.error, "expects an error string"); - }); + .expect(testData.expectError); }); it("handles a search", () => { return request(app) From 38db9e7dd68421f7124b0e3f4af6a33ae5cf42ab Mon Sep 17 00:00:00 2001 From: ionous Date: Sat, 31 Jan 2026 12:10:01 -0800 Subject: [PATCH 2/7] comments --- app/test/search_test.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/test/search_test.js b/app/test/search_test.js index ce3a8cb4..5dde4986 100644 --- a/app/test/search_test.js +++ b/app/test/search_test.js @@ -62,6 +62,7 @@ describe("searching for events", () => { .expect(200) .expect('Content-Type', /json/) .then(res => { + // pagination: still 14 events available; but we've asked for two at a time. assert.equal(res.body?.pagination.fullcount, 14); assert.equal(res.body?.pagination.offset, 0); assert.equal(res.body?.pagination.limit, 2); @@ -83,6 +84,7 @@ describe("searching for events", () => { .expect(200) .expect('Content-Type', /json/) .then(res => { + // pagination: still 14 events available; but we've asked for two at a time. assert.equal(res.body?.pagination?.fullcount, 14); assert.equal(res.body?.pagination?.offset, 2); assert.equal(res.body?.pagination?.limit, 2); From 2e1eec45ce0db6b3bd4d3826f770b542de7e8771 Mon Sep 17 00:00:00 2001 From: ionous Date: Sat, 31 Jan 2026 12:10:19 -0800 Subject: [PATCH 3/7] remove unused import statements --- app/appEndpoints.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/appEndpoints.js b/app/appEndpoints.js index 2b215726..1bd87e4b 100644 --- a/app/appEndpoints.js +++ b/app/appEndpoints.js @@ -2,8 +2,6 @@ const express = require('express'); const config = require("./config"); const errors = require("./util/errors"); const nunjucks = require("./nunjucks"); -const { initMail } = require("./emailer"); -const knex = require("./db"); // initialize on startup const app = express(); // shift.conf for nginx sets the x-forward header @@ -70,5 +68,5 @@ app.use(function(err, req, res, next) { console.error(err.stack); }); -// for testing +// for starting a server or running tests module.exports = app; From a274f6752e666bfba3860b882cf4c977605ee471 Mon Sep 17 00:00:00 2001 From: ionous Date: Thu, 19 Feb 2026 19:11:15 -0800 Subject: [PATCH 4/7] fix test data generation so its not subject to race conditions previously, since the test data was generated as it wrote to the database, different timings of the database writes could generate different data. this generates all the data first, and then writes that data. i also separated the data logging -- which cleans up the code slightly -- the command line make fake data can log without having to worry about the tests generating logs ( and mucking up the test output ) --- app/test/fakeData.js | 60 +++++++------ app/test/testdb.js | 7 +- app/test/testing.md | 184 ++++++++++++++++++++-------------------- tools/makeFakeEvents.js | 10 ++- 4 files changed, 137 insertions(+), 124 deletions(-) diff --git a/app/test/fakeData.js b/app/test/fakeData.js index e1988a81..92c7daa7 100644 --- a/app/test/fakeData.js +++ b/app/test/fakeData.js @@ -9,23 +9,32 @@ const password = "supersecret"; // number of days within which to create fake events const fakeRange = 7; -// promise an array of events +// promise an array of {event: {}, days: [{}]} // - firstDay is a dt day // - numEvents: a number of events to create // - seed: an optional random seed -function makeFakeData(firstDay, lastDay, numEvents) { - const fakeData = generateFakeData(firstDay, lastDay, numEvents); +function insertFakeData(fakeData) { + return db.query.transaction(tx => _insertData(tx, fakeData)); +} + +// +function logFakeData(data) { + data.forEach(({event, days}) => { + const url = config.site.url("addevent", `edit-${event.id}-${event.password}`); + const start = dt.friendlyDate(days[0].eventdate); + console.log(`created ${url}\n "${event.title}" with ${days.length} days starting on ${start}`); + }); +} + +function _insertData(tx, fakeData) { const promisedEvents = fakeData.map(data => { - return db.query('calevent') + return tx('calevent') .insert(data.event).then(row => { - const id = row[0]; // the magic to get the event id. - // when passing a seed (ex. for tests); silence the output. - if (!config.isTesting) { - logData(id, data); - } + const id = row[0]; // the magic to get the event id. + data.event.id = id; // update the data so we can return it. const promisedDays = data.days.map(at => { at.id = id; // assign the id before adding to the db - return db.query('caldaily').insert(at); + return tx('caldaily').insert(at); }); return Promise.all(promisedDays); }); @@ -34,29 +43,29 @@ function makeFakeData(firstDay, lastDay, numEvents) { return Promise.all(promisedEvents); } -// -function logData(id, data) { - const url = config.site.url("addevent", `edit-${id}-${password}`); - const start = dt.friendlyDate(data.days[0].eventdate); - console.log(`created "${data.event.title}" with ${data.days.length} days starting on ${start}\n ${url}`); -} - // build the data before inserting into the db // this avoids race conditions influencing the data generated. -function generateFakeData(firstDay, lastDay, numEvents) { +// nextEventId can be a number, in which case it creates events starting with that id. +function generateFakeData(firstDay, lastDay, numEvents, seed, nextEventId = undefined) { + faker.seed(seed); const out = []; + let nextDayId = nextEventId ? 1 : undefined; for (let i = 0; i< numEvents; i++) { // always have one event on the day specified; // on subsequent events, pick from a range of days. - const start = !i ? firstDay: + const start = !i ? firstDay : dt.convert(faker.date.between({ from: firstDay.toDate(), to: lastDay.toDate(), })); const title = faker.music.songName(); - const event = makeCalEvent(title); + const event = makeCalEvent(title, nextEventId); const numDays = randomDayCount(); - const days = makeCalDailies(start, numDays); + const days = makeCalDailies(start, numDays, nextDayId); + if (nextEventId !== undefined) { + nextEventId += 1; + nextDayId += numDays; + } out.push({ event, days, @@ -66,7 +75,7 @@ function generateFakeData(firstDay, lastDay, numEvents) { } // export! -module.exports = { makeFakeData }; +module.exports = { insertFakeData, generateFakeData, logFakeData }; function randomDayCount() { // some dumb weighted random @@ -88,7 +97,7 @@ function nextDay(days, refDate) { }); } -function makeCalDailies(start, numDays) { +function makeCalDailies(start, numDays, nextPkid = undefined) { const out = []; const active = faker.datatype.boolean(0.8); const flash = faker.datatype.boolean(!active? 0.8: 0.3); @@ -103,6 +112,7 @@ function makeCalDailies(start, numDays) { eventdate : db.toDate(start), eventstatus : active? EventStatus.Active : EventStatus.Cancelled, newsflash : msg, + pkid : (nextPkid !== undefined) ? (nextPkid++) : undefined, }); } return out; @@ -113,7 +123,7 @@ function capitalize(str, yes= true) { return (yes? first.toUpperCase() : first.toLowerCase() ) + str.slice(1); } -function makeCalEvent(title) { +function makeCalEvent(title, predefinedId = undefined) { const firstName = faker.person.firstName(); const lastName = faker.person.lastName(); const organizer = faker.person.fullName({firstName, lastName}); @@ -165,7 +175,7 @@ function makeCalEvent(title) { // created, // modified, changes, - // id: eventId, + id: predefinedId, name: organizer, email, hideemail, diff --git a/app/test/testdb.js b/app/test/testdb.js index c9b318c2..d1f85679 100644 --- a/app/test/testdb.js +++ b/app/test/testdb.js @@ -4,7 +4,7 @@ const dt = require("../util/dateTime"); const { faker } = require('@faker-js/faker'); const testData = require("./testData"); const db = require("../db"); -const { makeFakeData } = require("./fakeData"); +const { generateFakeData, insertFakeData } = require("./fakeData"); module.exports = { // generates a hand rolled set of data @@ -23,8 +23,9 @@ module.exports = { const firstDay = dt.fromYMDString("2002-08-01"); const lastDay = dt.fromYMDString("2002-08-31"); const numEvents = 46; - faker.seed(23204); // keeps the generated data stable. - await makeFakeData(firstDay, lastDay, numEvents); + const seed = 23204; // keeps the generated data stable. + const fakeData = generateFakeData(firstDay, lastDay, numEvents, seed, seed); + return insertFakeData(fakeData); }, destroy() { // leaves the tables in place; lets create drop them when needed. diff --git a/app/test/testing.md b/app/test/testing.md index 4d3313bb..93b0831a 100644 --- a/app/test/testing.md +++ b/app/test/testing.md @@ -17,95 +17,95 @@ By default tests use sqlite, you can test against mysql as well: `npm test -db= fakeData.js generates the following events: -* "The Tracks of My Tears" with 1 days starting on 2002-08-01 - http://localhost:3080/addevent/edit-1-supersecret -* "Knock On Wood" with 1 days starting on 2002-08-20 - http://localhost:3080/addevent/edit-2-supersecret -* "Tonight's the Night (Gonna Be Alright)" with 2 days starting on 2002-08-02 - http://localhost:3080/addevent/edit-3-supersecret -* "One" with 2 days starting on 2002-08-02 - http://localhost:3080/addevent/edit-4-supersecret -* "Whip It" with 4 days starting on 2002-08-16 - http://localhost:3080/addevent/edit-5-supersecret -* "Losing My Religion" with 5 days starting on 2002-08-22 - http://localhost:3080/addevent/edit-6-supersecret -* "I'm a Believer" with 1 days starting on 2002-08-20 - http://localhost:3080/addevent/edit-7-supersecret -* "Hips don't lie" with 1 days starting on 2002-08-28 - http://localhost:3080/addevent/edit-8-supersecret -* "Living For the City" with 1 days starting on 2002-08-01 - http://localhost:3080/addevent/edit-9-supersecret -* "Shake Down" with 1 days starting on 2002-08-08 - http://localhost:3080/addevent/edit-10-supersecret -* "Wicked Game" with 3 days starting on 2002-08-19 - http://localhost:3080/addevent/edit-11-supersecret -* "Jive Talkin'" with 2 days starting on 2002-08-28 - http://localhost:3080/addevent/edit-12-supersecret -* "Wheel of Fortune" with 3 days starting on 2002-08-17 - http://localhost:3080/addevent/edit-13-supersecret -* "Travellin' Band" with 1 days starting on 2002-08-09 - http://localhost:3080/addevent/edit-14-supersecret -* "Bye" with 1 days starting on 2002-08-07 - http://localhost:3080/addevent/edit-15-supersecret -* "The Girl From Ipanema" with 2 days starting on 2002-08-26 - http://localhost:3080/addevent/edit-16-supersecret -* "If (They Made Me a King)" with 2 days starting on 2002-08-15 - http://localhost:3080/addevent/edit-17-supersecret -* "This Used to Be My Playground" with 2 days starting on 2002-08-16 - http://localhost:3080/addevent/edit-18-supersecret -* "Crying" with 5 days starting on 2002-08-27 - http://localhost:3080/addevent/edit-19-supersecret -* "Na Na Hey Hey (Kiss Him Goodbye)" with 2 days starting on 2002-08-13 - http://localhost:3080/addevent/edit-20-supersecret -* "Upside Down" with 1 days starting on 2002-08-28 - http://localhost:3080/addevent/edit-21-supersecret -* "Love Me Do" with 4 days starting on 2002-08-20 - http://localhost:3080/addevent/edit-22-supersecret -* "Breathe" with 5 days starting on 2002-08-09 - http://localhost:3080/addevent/edit-23-supersecret -* "Brandy (You're A Fine Girl)" with 2 days starting on 2002-08-23 - http://localhost:3080/addevent/edit-24-supersecret -* "Swanee" with 2 days starting on 2002-08-16 - http://localhost:3080/addevent/edit-25-supersecret -* "Earth Angel" with 1 days starting on 2002-08-16 - http://localhost:3080/addevent/edit-26-supersecret -* "Let's Get it On" with 1 days starting on 2002-08-25 - http://localhost:3080/addevent/edit-27-supersecret -* "Arthur's Theme (Best That You Can Do)" with 1 days starting on 2002-08-08 - http://localhost:3080/addevent/edit-28-supersecret -* "Sunday" with 1 days starting on 2002-08-25 - http://localhost:3080/addevent/edit-29-supersecret -* "Nothing's Gonna Stop Us Now" with 2 days starting on 2002-08-25 - http://localhost:3080/addevent/edit-30-supersecret -* "Change the World" with 4 days starting on 2002-08-01 - http://localhost:3080/addevent/edit-31-supersecret -* "Tammy" with 2 days starting on 2002-08-10 - http://localhost:3080/addevent/edit-32-supersecret -* "Come Together" with 2 days starting on 2002-08-10 - http://localhost:3080/addevent/edit-33-supersecret -* "Take On Me" with 2 days starting on 2002-08-03 - http://localhost:3080/addevent/edit-34-supersecret -* "Fantasy" with 1 days starting on 2002-08-10 - http://localhost:3080/addevent/edit-35-supersecret -* "Centerfold" with 5 days starting on 2002-08-02 - http://localhost:3080/addevent/edit-36-supersecret -* "I Gotta Feeling" with 2 days starting on 2002-08-18 - http://localhost:3080/addevent/edit-37-supersecret -* "I Can't Get Started" with 2 days starting on 2002-08-19 - http://localhost:3080/addevent/edit-38-supersecret -* "Only The Lonely (Know The Way I Feel)" with 2 days starting on 2002-08-27 - http://localhost:3080/addevent/edit-39-supersecret -* "Escape (The Pina Colada Song)" with 1 days starting on 2002-08-07 - http://localhost:3080/addevent/edit-40-supersecret -* "(Ghost) Riders in the Sky" with 2 days starting on 2002-08-04 - http://localhost:3080/addevent/edit-41-supersecret -* "When a Man Loves a Woman" with 3 days starting on 2002-08-24 - http://localhost:3080/addevent/edit-42-supersecret -* "Dreamlover" with 1 days starting on 2002-08-27 - http://localhost:3080/addevent/edit-43-supersecret -* "Brown Eyed Girl" with 3 days starting on 2002-08-20 - http://localhost:3080/addevent/edit-44-supersecret -* "(They Long to Be) Close to You" with 1 days starting on 2002-08-17 - http://localhost:3080/addevent/edit-45-supersecret -* "Rock With You" with 1 days starting on 2002-08-24 - http://localhost:3080/addevent/edit-46-supersecret \ No newline at end of file +* http://localhost:3080/addevent/edit-23204-supersecret + "The Tracks of My Tears" with 1 days starting on Thu, Aug 1st +* http://localhost:3080/addevent/edit-23205-supersecret + "Knock On Wood" with 1 days starting on Tue, Aug 20th +* http://localhost:3080/addevent/edit-23206-supersecret + "Tonight's the Night (Gonna Be Alright)" with 2 days starting on Fri, Aug 2nd +* http://localhost:3080/addevent/edit-23207-supersecret + "One" with 2 days starting on Fri, Aug 2nd +* http://localhost:3080/addevent/edit-23208-supersecret + "Whip It" with 4 days starting on Fri, Aug 16th +* http://localhost:3080/addevent/edit-23209-supersecret + "Losing My Religion" with 5 days starting on Thu, Aug 22nd +* http://localhost:3080/addevent/edit-23210-supersecret + "I'm a Believer" with 1 days starting on Tue, Aug 20th +* http://localhost:3080/addevent/edit-23211-supersecret + "Hips don't lie" with 1 days starting on Wed, Aug 28th +* http://localhost:3080/addevent/edit-23212-supersecret + "Living For the City" with 1 days starting on Thu, Aug 1st +* http://localhost:3080/addevent/edit-23213-supersecret + "Shake Down" with 1 days starting on Thu, Aug 8th +* http://localhost:3080/addevent/edit-23214-supersecret + "Wicked Game" with 3 days starting on Mon, Aug 19th +* http://localhost:3080/addevent/edit-23215-supersecret + "Jive Talkin'" with 2 days starting on Wed, Aug 28th +* http://localhost:3080/addevent/edit-23216-supersecret + "Wheel of Fortune" with 3 days starting on Sat, Aug 17th +* http://localhost:3080/addevent/edit-23217-supersecret + "Travellin' Band" with 1 days starting on Fri, Aug 9th +* http://localhost:3080/addevent/edit-23218-supersecret + "Bye" with 1 days starting on Wed, Aug 7th +* http://localhost:3080/addevent/edit-23219-supersecret + "The Girl From Ipanema" with 2 days starting on Mon, Aug 26th +* http://localhost:3080/addevent/edit-23220-supersecret + "If (They Made Me a King)" with 2 days starting on Thu, Aug 15th +* http://localhost:3080/addevent/edit-23221-supersecret + "This Used to Be My Playground" with 2 days starting on Fri, Aug 16th +* http://localhost:3080/addevent/edit-23222-supersecret + "Crying" with 5 days starting on Tue, Aug 27th +* http://localhost:3080/addevent/edit-23223-supersecret + "Na Na Hey Hey (Kiss Him Goodbye)" with 2 days starting on Tue, Aug 13th +* http://localhost:3080/addevent/edit-23224-supersecret + "Upside Down" with 1 days starting on Wed, Aug 28th +* http://localhost:3080/addevent/edit-23225-supersecret + "Love Me Do" with 4 days starting on Tue, Aug 20th +* http://localhost:3080/addevent/edit-23226-supersecret + "Breathe" with 5 days starting on Fri, Aug 9th +* http://localhost:3080/addevent/edit-23227-supersecret + "Brandy (You're A Fine Girl)" with 2 days starting on Fri, Aug 23rd +* http://localhost:3080/addevent/edit-23228-supersecret + "Swanee" with 2 days starting on Fri, Aug 16th +* http://localhost:3080/addevent/edit-23229-supersecret + "Earth Angel" with 1 days starting on Fri, Aug 16th +* http://localhost:3080/addevent/edit-23230-supersecret + "Let's Get it On" with 1 days starting on Sun, Aug 25th +* http://localhost:3080/addevent/edit-23231-supersecret + "Arthur's Theme (Best That You Can Do)" with 1 days starting on Thu, Aug 8th +* http://localhost:3080/addevent/edit-23232-supersecret + "Sunday" with 1 days starting on Sun, Aug 25th +* http://localhost:3080/addevent/edit-23233-supersecret + "Nothing's Gonna Stop Us Now" with 2 days starting on Sun, Aug 25th +* http://localhost:3080/addevent/edit-23234-supersecret + "Change the World" with 4 days starting on Thu, Aug 1st +* http://localhost:3080/addevent/edit-23235-supersecret + "Tammy" with 2 days starting on Sat, Aug 10th +* http://localhost:3080/addevent/edit-23236-supersecret + "Come Together" with 2 days starting on Sat, Aug 10th +* http://localhost:3080/addevent/edit-23237-supersecret + "Take On Me" with 2 days starting on Sat, Aug 3rd +* http://localhost:3080/addevent/edit-23238-supersecret + "Fantasy" with 1 days starting on Sat, Aug 10th +* http://localhost:3080/addevent/edit-23239-supersecret + "Centerfold" with 5 days starting on Fri, Aug 2nd +* http://localhost:3080/addevent/edit-23240-supersecret + "I Gotta Feeling" with 2 days starting on Sun, Aug 18th +* http://localhost:3080/addevent/edit-23241-supersecret + "I Can't Get Started" with 2 days starting on Mon, Aug 19th +* http://localhost:3080/addevent/edit-23242-supersecret + "Only The Lonely (Know The Way I Feel)" with 2 days starting on Tue, Aug 27th +* http://localhost:3080/addevent/edit-23243-supersecret + "Escape (The Pina Colada Song)" with 1 days starting on Wed, Aug 7th +* http://localhost:3080/addevent/edit-23244-supersecret + "(Ghost) Riders in the Sky" with 2 days starting on Sun, Aug 4th +* http://localhost:3080/addevent/edit-23245-supersecret + "When a Man Loves a Woman" with 3 days starting on Sat, Aug 24th +* http://localhost:3080/addevent/edit-23246-supersecret + "Dreamlover" with 1 days starting on Tue, Aug 27th +* http://localhost:3080/addevent/edit-23247-supersecret + "Brown Eyed Girl" with 3 days starting on Tue, Aug 20th +* http://localhost:3080/addevent/edit-23248-supersecret + "(They Long to Be) Close to You" with 1 days starting on Sat, Aug 17th +* http://localhost:3080/addevent/edit-23249-supersecret + "Rock With You" with 1 days starting on Sat, Aug 24th diff --git a/tools/makeFakeEvents.js b/tools/makeFakeEvents.js index 99fd0d91..e9bed89a 100644 --- a/tools/makeFakeEvents.js +++ b/tools/makeFakeEvents.js @@ -2,9 +2,9 @@ * create one or more fake events. * ex. npm run -w tools make-fake-events */ -const knex = require("shift-docs/db"); +const db = require("shift-docs/db"); const dt = require("shift-docs/util/dateTime"); -const { makeFakeData } = require("shift-docs/test/fakeData"); +const { generateFakeData, insertFakeData, logFakeData } = require("shift-docs/test/fakeData"); // todo: improve commandline parsing // this uses npm's command vars ( probably a bad idea ) @@ -31,10 +31,12 @@ async function makeFakeEvents() { const firstDay = dt.getNow().add(args.start, 'days'); const lastDay = firstDay.add(args.range, 'days'); const numEvents = args.make; - return knex.initialize().then(_ => { - return makeFakeData(firstDay, lastDay, numEvents); + const fakeData = generateFakeData(firstDay, lastDay, numEvents); + return db.initialize().then(_ => { + return insertFakeData(fakeData); }) .then(_ => { + logFakeData(fakeData); console.log("done"); // can't use top-level "await" with commonjs modules // ( ie. await makeFakeEvents() ) From 3beaed3ad502ebf7d509a8419d34901661bc0f0e Mon Sep 17 00:00:00 2001 From: ionous Date: Thu, 19 Feb 2026 19:34:23 -0800 Subject: [PATCH 5/7] update comments --- app/test/fakeData.js | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/app/test/fakeData.js b/app/test/fakeData.js index 92c7daa7..0f8b648d 100644 --- a/app/test/fakeData.js +++ b/app/test/fakeData.js @@ -9,23 +9,12 @@ const password = "supersecret"; // number of days within which to create fake events const fakeRange = 7; -// promise an array of {event: {}, days: [{}]} -// - firstDay is a dt day -// - numEvents: a number of events to create -// - seed: an optional random seed +// writes an array of the fake data format into the db. +// separating generation from insertion prevents race conditions from influencing the generated data. function insertFakeData(fakeData) { return db.query.transaction(tx => _insertData(tx, fakeData)); } -// -function logFakeData(data) { - data.forEach(({event, days}) => { - const url = config.site.url("addevent", `edit-${event.id}-${event.password}`); - const start = dt.friendlyDate(days[0].eventdate); - console.log(`created ${url}\n "${event.title}" with ${days.length} days starting on ${start}`); - }); -} - function _insertData(tx, fakeData) { const promisedEvents = fakeData.map(data => { return tx('calevent') @@ -43,8 +32,19 @@ function _insertData(tx, fakeData) { return Promise.all(promisedEvents); } -// build the data before inserting into the db -// this avoids race conditions influencing the data generated. +// prints an array of fake data format to console. +function logFakeData(data) { + data.forEach(({event, days}) => { + const url = config.site.url("addevent", `edit-${event.id}-${event.password}`); + const start = dt.friendlyDate(days[0].eventdate); + console.log(`created ${url}\n "${event.title}" with ${days.length} days starting on ${start}`); + }); +} + +// promise an array of {event: {}, days: [{}]} +// - firstDay is a dt day +// - numEvents: a number of events to create +// - seed: an optional random seed // nextEventId can be a number, in which case it creates events starting with that id. function generateFakeData(firstDay, lastDay, numEvents, seed, nextEventId = undefined) { faker.seed(seed); From 38d8aa4dfd4a4ea6e626235025641c8635e5a01c Mon Sep 17 00:00:00 2001 From: ionous Date: Thu, 19 Mar 2026 17:55:20 -0700 Subject: [PATCH 6/7] use a custom test runner to handle command line params --- .vscode/launch.json | 6 ++-- app/config.js | 69 +++++++++++++++++++++++++++++--------- app/db.js | 5 ++- app/package.json | 2 +- app/test/db_setup.js | 42 ++++++++++++----------- app/test/testRunner.js | 74 +++++++++++++++++++++++++++++++++++++++++ app/test/testdb.js | 4 +-- app/test/testing.md | 9 ++--- app/util/cmdLine.js | 64 +++++++++++++++++++++++++++++++++++ package.json | 2 +- tools/makeFakeEvents.js | 2 +- 11 files changed, 230 insertions(+), 49 deletions(-) create mode 100644 app/test/testRunner.js create mode 100644 app/util/cmdLine.js diff --git a/.vscode/launch.json b/.vscode/launch.json index 57440f73..e51e057b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,7 +5,7 @@ "version": "0.2.0", "configurations": [ { - "name": "Launch via NPM", + "name": "Node Server", "request": "launch", "runtimeArgs": [ "run-script", @@ -18,7 +18,7 @@ "type": "node" }, { - "name": "Test", + "name": "Full Test", "request": "launch", "runtimeArgs": [ "run-script", @@ -29,6 +29,6 @@ "/**" ], "type": "node" - }, + } ] } \ No newline at end of file diff --git a/app/config.js b/app/config.js index 3934b2e8..af4fb504 100644 --- a/app/config.js +++ b/app/config.js @@ -1,10 +1,34 @@ +// global configuration for the backend +// holds all environment, commandline options, paths, etc. +// +// const config = require('./config'); +// const path = require('path'); const fs = require("fs"); +const { CommandLine } = require('./util/cmdLine.js'); +// helper to read environment variables function env_default(field, def) { return process.env[field] ?? def; } +// helper to turn lines of quoted text into a single string +function lines(...lines) { + return lines.join("\n"); +} + +const cmdLine = new CommandLine({ + db: lines( + `an optional string. can be 'sqlite' or 'mysql'.`, + `the behavior depends on context. for tests, defaults to sqlite. for prod and dev defaults to mysql.`, + `- 'mysql': requires docker. for prod and dev pulls its full configuration from the environment; tests have a hardcoded configuration.`, + `- 'sqlite': can followed by the path to its file. For example: '-db=sqlite:../bin/events.db'. That example path is its default.`, + ), + db_debug: lines( + `an optional flag. adds extra debugging info for queries.`, + ), +}); + // node server listen for its http requests // fix? there is no such environment variable right now. const listen = env_default('NODE_PORT', 3080); @@ -15,23 +39,26 @@ const siteHost = siteUrl(listen); // location of app.js ( same as config.js ) const appPath = path.resolve(__dirname); -// for max file size +// for max image size const bytesPerMeg = 1024*1024; const staticFiles = env_default('SHIFT_STATIC_FILES'); -const isTesting = !!(process.env.npm_lifecycle_event || "").match(/test$/); -// read the command line parameter for db configuration -const dbType = env_default('npm_config_db'); -const dbDebug = !!env_default('npm_config_db_debug'); +// running node --test with a glob ( ex. node --test **/*_test.js ) +// injects all of those files onto the command line as separate arguments. +const isTesting = process.env.npm_lifecycle_event && process.env.npm_lifecycle_event.startsWith('test'); +// ^ FIX: check for --test if running node directly? + +const dbConfig = getDatabaseConfig(cmdLine.options.db, cmdLine.bool('db_debug'), isTesting); +// The config object exported const config = { appPath, api: { header: 'Api-Version', version: "3.59.10", }, - db: getDatabaseConfig(dbType, isTesting), + db: dbConfig, // maybe bad, but some code likes to know: isTesting, // a nodemailer friendly config, or false if smtp is not configured. @@ -166,7 +193,7 @@ function getSmtpSettings() { if (emailCfg) { try { const raw= fs.readFileSync(emailCfg, "utf8"); - return JSON.parse(raw); + return JSON.getBool(raw); } catch (err) { // its okay if there is no such file... if (err.code !== 'ENOENT') { @@ -195,17 +222,27 @@ function getSmtpSettings() { } } -// our semi-agnostic database configuration -function getDatabaseConfig(dbType, isTesting) { - // dbType comes from the command-line - // if nothing was specfied, use the MYSQL_DATABASE environment variable - const env = env_default('MYSQL_DATABASE') - if (!dbType && env) { - dbType = env.startsWith("sqlite") ? env : null; - } +// nothing specified in the env fails +// sqlite specified in the env uses.... sqlite. +// anything other than sqlite in the env is assumed to be a mysql spec. +function readDbEnv(env = 'MYSQL_DATABASE') { + const dbType = env_default(env); if (!dbType) { - dbType = isTesting ? 'sqlite' : 'mysql' + throw new Error(`expected db type on command line or '${env}' env variable`); } + return dbType.startsWith("sqlite") ? dbType : 'mysql'; +} + +// returns our semi-agnostic database configuration. +// read the command line and environment for the desired db. +// +// dbType can be 'sqlite' or 'mysql' or none. +// 'mysql' pulls its full configuration from the environment. +// 'sqlite' can optionally followed by the path to its file. +// testing defaults to sqlite; dev and prod fallback to the environment. +// +function getDatabaseConfig(dbType, dbDebug, isTesting) { + dbType = dbType || (isTesting ? 'sqlite' : readDbEnv()); const [name, parts] = dbType.split(':'); const config = { mysql: !isTesting ? getMysqlDefault : getMysqlTesting, diff --git a/app/db.js b/app/db.js index 991cbb89..b0d4912e 100644 --- a/app/db.js +++ b/app/db.js @@ -7,6 +7,7 @@ const dt = require("./util/dateTime"); const dbConfig = unpackConfig(config.db); const useSqlite = config.db.type === 'sqlite'; const dropOnCreate = config.db.connect?.name === 'shift_test'; +let prevSetup; const db = { config: config.db, @@ -19,10 +20,11 @@ const db = { // waits to open a connection. async initialize(name) { if (db.query) { - throw new Error("db already initialized"); + throw new Error(`db being initialized by ${name} when already initialized by ${this.initialized}.`); } const connection = knex(dbConfig); db.query = connection; + db.initialized = name; await connection; }, @@ -33,6 +35,7 @@ const db = { throw new Error("db already destroyed"); } db.query = false; + db.initialized = false; return connection.destroy(); }, diff --git a/app/package.json b/app/package.json index 2bad03e3..8f70c937 100644 --- a/app/package.json +++ b/app/package.json @@ -20,7 +20,7 @@ }, "scripts": { "start": "node app.js", - "test": "node --test --trace-warnings --test-global-setup=test/db_setup.js --test-concurrency=1 **/*_test.js --" + "test": "node ./test/testRunner.js" }, "devDependencies": { "shelljs": "^0.10.0", diff --git a/app/test/db_setup.js b/app/test/db_setup.js index bbeff445..97089953 100644 --- a/app/test/db_setup.js +++ b/app/test/db_setup.js @@ -1,25 +1,21 @@ +// Configures mysql while testing. // // > npm test -db=mysql -db_debug=true // -const shell = require( 'shelljs'); // cross-platform shell interaction -const { setTimeout } = require('node:timers/promises'); -const db = require('../db'); +const shell = require('shelljs'); // talks to docker in a cross-platform way. +const { setTimeout } = require('node:timers/promises'); // mysql takes time to start. +const db = require('../db'); -// re: Unknown cli config "--db". This will stop working in the next major version of npm. -// the alternative is ugly; tbd but would rather wait till it breaks. -// const custom = process.argv.indexOf("--"); -// if (custom >= 0) { -// console.log(process.argv.slice(custom+1)); -// } +const dockerImage = `mysql:8.4.7`; // fix? pull from env somewhere. function shutdownMysql() { - console.log(`shutting down up docker mysql...`); + console.log(`shutting down up docker...`); const code = shell.exec("docker stop test_shift", {silent: true}).code; - console.log(`docker shutdown ${code}.`); + console.log(`docker shutdown ${code === 0 ? 'successfully' : code}.`); } async function startupMysql(connect) { - console.log(`setting up docker mysql...`); + console.log(`setting up docker image ${dockerImage}...`); // ex. if a test failed and the earlier run didn't exit well. const alreadyExists = shell.exec("docker start test_shift", {silent: true}).code === 0; if (alreadyExists) { @@ -27,21 +23,21 @@ async function startupMysql(connect) { console.log("docker test_shift already running. reusing the container."); } - // setup docker mysql + // setup docker mysql for testing // https://dev.mysql.com/doc/refman/8.4/en/docker-mysql-more-topics.html // https://docs.docker.com/reference/cli/docker/container/run/ // https://hub.docker.com/_/mysql/ - if (!alreadyExists && shell.exec(lines( + if (!alreadyExists && shell.exec(join( `docker run`, `--name test_shift`, // container name `--detach`, // keep running the container in the background `--rm` , // cleanup the container after exit - `-p ${connect.port}:3306`, // expose mysql's internal 3306 port as our custom port + `-p ${connect.port}:3306`, // expose mysql's internal 3306 port as our custom port `-e MYSQL_RANDOM_ROOT_PASSWORD=true`, // alt: MYSQL_ROOT_PASSWORD `-e MYSQL_DATABASE=${connect.name}`, `-e MYSQL_USER=${connect.user}`, `-e MYSQL_PASSWORD=${connect.pass}`, - `mysql:8.4.7`, // fix? pull from env somewhere. + dockerImage, `--disable-log-bin=true`, `--character-set-server=utf8mb4`, `--collation-server=utf8mb4_unicode_ci`)).code !== 0) { @@ -51,7 +47,7 @@ async function startupMysql(connect) { } async function initConnection() { - // configure the connection + // configure a connection await db.initialize("test setup"); // wait for an empty query to succeed. @@ -70,25 +66,31 @@ async function initConnection() { console.log(`waiting ${i} seconds....`); await setTimeout(i * 1000); } + + await db.destroy(); } -// helpers +// helper to read environment variables function env_default(field, def) { return process.env[field] ?? def; } -function lines(...lines) { + +// helper to turn lines of quoted text into a single string +function join(...lines) { return lines.join(" "); } +// when testing, called once before any tests start async function globalSetup() { + console.log("global setup using: ", JSON.stringify(db.config, null, " ")); if (db.config.type === 'mysql') { await startupMysql(db.config.connect); } await initConnection(); } +// when testing, called once after all tests have completed async function globalTeardown() { - await db.destroy(); if (db.config.type === 'mysql') { shutdownMysql(); } diff --git a/app/test/testRunner.js b/app/test/testRunner.js new file mode 100644 index 00000000..24a92e21 --- /dev/null +++ b/app/test/testRunner.js @@ -0,0 +1,74 @@ +// +// A custom wrapper for running multiple tests. +// +// Rationale: node's test runner ignores multiple files when using its only and pattern filters. +// https://github.com/nodejs/node/issues/51384 +// +const fs = require('fs'); // search for test files +const path = require('path'); // building paths to run those files +const shell = require('shelljs'); // runs node from node; what could be simpler :sob:. +const { CommandLine } = require('../util/cmdLine.js'); +const { globalSetup, globalTeardown } = require('./db_setup.js'); + +const cmdLine = new CommandLine({ + only: `flag to invoke --test-only`, + pattern: `regex for --test-name-pattern`, +}); + +// arguments to pass to node for each test +const testArgs = [ + `--trace-warnings`, + `--test-concurrency=1`, + // normally we'd let node handle the global setup ( ex. starting docker ) + // however, when running multiple tests, the setup should only happen once. + //`--test-global-setup=test/db_setup.js` +]; + +// the custom test runner +async function runTests() { + const startingDir = process.cwd(); // ex. /Users/ionous/dev/shift/real-shift/app + const originalCmdLine = process.argv.slice(2); // skip 0 (bin/node) and 1 (testRunner.js) + await globalSetup(); + _runTests(startingDir, testArgs, originalCmdLine, cmdLine.options); + await globalTeardown(); +} +runTests(); + +// helpers: +function _runTests(dir, args, orig, { only, pattern }) { + if (only && pattern) { + throw new Error(`The test runner expects at most one option. Both only and pattern were specified.`); + } + const files = findTestFiles(dir); + if (!only && !pattern) { + // the regular test command can handle processing multiple files + test([`--test`].concat(args, files, '--', orig)); + + } else if (only) { + // find will stop running tests when something returns a non-zero error code. + files.find(f => test([`--test-only`].concat(args, f,'--', orig)) !== 0 ); + + } else if (pattern) { + // find will stop running tests when something returns a non-zero error code. + files.find(f => test(args.concat(`--test-name-pattern="${pattern}"`, f)) !== 0 ); + } +} + +// return an array of filenames to test +// (returned paths are relative to dir) +function findTestFiles(dir) { + const entries = fs.readdirSync(dir, { + withFileTypes: true, + recursive: true, + }); + return entries.filter(f => f.name.endsWith("_test.js")) + .map(f => path.relative(dir, path.resolve(f.parentPath, f.name))); +} + +function test(parts) { + const cmdLine = `node ${parts.join(' ')}`; + shell.echo(cmdLine); + // note: shell.cmd is safer, but i can't get it to work with quoted text options. :shrug: + // ex. --test-name-pattern="date time" looks correct when echo'd but doesn't pass the pattern to node. + return shell.exec(cmdLine).code; +} \ No newline at end of file diff --git a/app/test/testdb.js b/app/test/testdb.js index d1f85679..89899197 100644 --- a/app/test/testdb.js +++ b/app/test/testdb.js @@ -9,7 +9,7 @@ const { generateFakeData, insertFakeData } = require("./fakeData"); module.exports = { // generates a hand rolled set of data setupTestData: async (name) => { - await db.initialize(); + await db.initialize('setupTestData'); await tables.dropTables(); await tables.createTables(); faker.seed(23204); // uses lorem generator @@ -17,7 +17,7 @@ module.exports = { }, // uses faker to generate a good amount of fake data setupFakeData: async (name) => { - await db.initialize(); + await db.initialize('setupFakeData'); await tables.dropTables(); await tables.createTables(); const firstDay = dt.fromYMDString("2002-08-01"); diff --git a/app/test/testing.md b/app/test/testing.md index 93b0831a..9f52484c 100644 --- a/app/test/testing.md +++ b/app/test/testing.md @@ -2,16 +2,17 @@ To test the backend, at the root of repo run: `npm test`. - All tests use the [Node Test runner](https://nodejs.org/docs/latest/api/test.html#test-runner), with [supertest](https://github.com/forwardemail/supertest) for making (local) http requests. ## Isolating tests: -Tests can be identified by name: ex. `npm test -- --test-name-pattern="ical feed"` +Tests can be singled out by name: ex. `npm test -- -pattern="ical"` + +Or, in the test code, they can be temporarily marked with 'only'. For example: `describe.only("ical feed")`, and then isolated with: `npm test -- -only` -Or, temporarily can be marked with 'only' in the code. For example: `describe.only()`, and then selected with: `npm test -- --test-only``` +# Mysql tests: -By default tests use sqlite, you can test against mysql as well: `npm test -db=mysql`. It launches a standalone docker container for the tests. Additionally, `npm test -db_debug` will log queries to the db. +By default tests use sqlite, you can test against mysql as well: `npm test -- -db=mysql`. It launches a standalone docker container for the tests. Additionally, `npm test -- -db_debug` will log queries to the db. # Test Data diff --git a/app/util/cmdLine.js b/app/util/cmdLine.js new file mode 100644 index 00000000..c833ef64 --- /dev/null +++ b/app/util/cmdLine.js @@ -0,0 +1,64 @@ + +// a simple helper to parse key=value parameters passed via the command line. +// adds the user specified values to a '.options' member +// ex. `-hello=world` becomes `.options.hello === 'world` +class CommandLine { + // pass valid command line options as a map of name to documentation. + // will throw if the user has specified an option that isn't part of the known set + constructor(known) { + const pairs = process.argv + .filter(arg => arg.startsWith('-') && !arg.startsWith('--')) + .map(arg => { + const match = arg.slice(1).match(/^(\w+)=(.+)|(\w+)$/); + // an argument '-a=b' gets split into key, value + // an argument '-c' gets assigned to flag + const [str, key, value, flag] = (match || [arg]); + // return them both as pairs to generate the options + return (key !== undefined) ? [key, value] : + (flag !== undefined) ? [flag, 'true'] : + [str]; + }); + // note: allows unknown options + // ( ex. so test runner and tests can have overlapping command lines ) + this.known = known; + this.options = Object.fromEntries(pairs); + } + // read a true/false style option. + // or undefined if no such option was specified. + bool(key) { + if (!(key in this.known)) { + console.error(`unknown key ${key}.`); + this.listKnown(this.known, console.error); + throw new Error("unexpected boolean key"); + } + const value = this.options[key]; + if (value) { + const map = { + '0': false, 'false': false, + '1': true, 'true': true, + }; + const res = map[value]; + if (res === undefined) { + console.error(`CommandLine had '${value}' when a boolean value for '${key}' was expected`); + throw new Error("unexpected boolean value"); + } + return res; + } + } +} + +function listKnown(known, out) { + out(`CommandLine options:`); + Object.keys(known).forEach(key => { + const value = known[key]; + out(`* ${key} = ${value}`); + }); +} + +function quote(list) { + return list.map(quote => `'${quote}'`).join(", ") +} + +module.exports = { + CommandLine +}; diff --git a/package.json b/package.json index 2e33008e..9df17ce8 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "postdeploy": "cp site/public/404/index.html site/public/404.html", "postinstall": "hugo-installer --version 0.144.2", - "test": "npm -w app test --", + "test": "npm run -w app test --", "preview": "concurrently --kill-others-on-fail \"npm:preview-*\"", "preview-hugo": "npm run dev-hugo", diff --git a/tools/makeFakeEvents.js b/tools/makeFakeEvents.js index e9bed89a..f8d2e5dc 100644 --- a/tools/makeFakeEvents.js +++ b/tools/makeFakeEvents.js @@ -32,7 +32,7 @@ async function makeFakeEvents() { const lastDay = firstDay.add(args.range, 'days'); const numEvents = args.make; const fakeData = generateFakeData(firstDay, lastDay, numEvents); - return db.initialize().then(_ => { + return db.initialize('makeFakeEvents').then(_ => { return insertFakeData(fakeData); }) .then(_ => { From 5790f67fb53c16071eb4583545a726db2f91f3f7 Mon Sep 17 00:00:00 2001 From: ionous Date: Thu, 19 Mar 2026 19:02:57 -0700 Subject: [PATCH 7/7] fix json call probably from a global find and replace from the command line parameters lookup --- app/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.js b/app/config.js index e54b2516..384e3d02 100644 --- a/app/config.js +++ b/app/config.js @@ -193,7 +193,7 @@ function getSmtpSettings() { if (emailCfg) { try { const raw= fs.readFileSync(emailCfg, "utf8"); - return JSON.getBool(raw); + return JSON.parse(raw); } catch (err) { // its okay if there is no such file... if (err.code !== 'ENOENT') {