diff --git a/.vscode/launch.json b/.vscode/launch.json index 2916fab..6e2d4aa 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,10 @@ "skipFiles": [ "/**" ], - "type": "node" + "type": "node", + "outFiles": [ + "${workspaceFolder}/dist/**/*.js" + ], }, { "type": "node", @@ -54,11 +57,11 @@ { "type": "node", "request": "launch", - "name": "Launch checkGithubRepos", + "name": "Launch checkReposScheduled", "skipFiles": [ "/**" ], - "program": "${workspaceFolder}/dist/checkGithubRepos.js", + "program": "${workspaceFolder}/dist/checkReposScheduled.js", "sourceMaps": true, "outFiles": [ "${workspaceFolder}/dist/**/*.js" @@ -79,11 +82,11 @@ { "type": "node", "request": "launch", - "name": "Launch checkUserTestRepos", + "name": "Launch checkReposTriggered", "skipFiles": [ "/**" ], - "program": "${workspaceFolder}/dist/checkUserTestRepos.js", + "program": "${workspaceFolder}/dist/checkReposTriggered.js", "sourceMaps": true, "outFiles": [ "${workspaceFolder}/dist/**/*.js" @@ -125,14 +128,14 @@ "false" ] }, - { + { "type": "node", "request": "launch", - "name": "Launch LSP checkGithubRepos", + "name": "Launch LSP checkReposScheduled", "skipFiles": [ "/**" ], - "program": "${workspaceFolder}/dist/checkGithubRepos.js", + "program": "${workspaceFolder}/dist/checkReposScheduled.js", "sourceMaps": true, "outFiles": [ "${workspaceFolder}/dist/**/*.js" @@ -151,5 +154,33 @@ ], "preLaunchTask": "npm: build" }, + { + "type": "node", + "request": "launch", + "name": "Launch Go checkReposTriggered", + "skipFiles": [ + "/**" + ], + "program": "${workspaceFolder}/dist/checkReposTriggered.js", + "sourceMaps": true, + "outFiles": [ + "${workspaceFolder}/dist/**/*.js" + ], + "args": [ + "tsserver", + "https://github.com/microsoft/typescript-go.git", + "main", + "3176", + "false", + "./artifacts/repos.json", + "1", + "1", + "RepoResults", + "true", + "n/a", + "false" + ], + "preLaunchTask": "npm: build" + }, ] } \ No newline at end of file diff --git a/README.md b/README.md index 3def5c2..63af24f 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,11 @@ To run online, you can These commands can also be run locally. ```sh -# New Error Detector (a.k.a. "git tests") -node dist/checkGithubRepos.js [post-results] [repo-count] [repo-start-index] [old-ts-version-on-npm] [old-ts-version-on-npm] +# New Error Detector (a.k.a. "scheduled tests") +node dist/checkReposScheduled.js [post-results] [repo-count] [repo-start-index] [old-ts-version-on-npm] [old-ts-version-on-npm] -# Inline User Test Reporter (a.k.a. "user tests") -node dist/checkUserTestRepos.js [post-results] [ts-repo-url] [head-ref] [requesting-user] [source-issue] [github-comment-id-for-updates] [query-repos-by-stars] +# Inline User Test Reporter (a.k.a. "triggered tests") +node dist/checkReposTriggered.js [post-results] [ts-repo-url] [head-ref] [requesting-user] [source-issue] [github-comment-id-for-updates] [query-repos-by-stars] ``` diff --git a/azure-pipelines-gitTests-lsp-js.yml b/azure-pipelines-gitTests-lsp-js.yml index cd7f62c..a0cbcc0 100644 --- a/azure-pipelines-gitTests-lsp-js.yml +++ b/azure-pipelines-gitTests-lsp-js.yml @@ -17,7 +17,8 @@ pool: extends: template: azure-pipelines-gitTests-template.yml parameters: - ENTRYPOINT: lsp + ENTRYPOINT: fuzzer OLD_VERSION: '0' NEW_VERSION: main LANGUAGE: JavaScript + REPO_IS_TSGO: true diff --git a/azure-pipelines-gitTests-lsp-ts.yml b/azure-pipelines-gitTests-lsp-ts.yml index d76cf50..0c22dd3 100644 --- a/azure-pipelines-gitTests-lsp-ts.yml +++ b/azure-pipelines-gitTests-lsp-ts.yml @@ -17,8 +17,9 @@ pool: extends: template: azure-pipelines-gitTests-template.yml parameters: - ENTRYPOINT: lsp + ENTRYPOINT: fuzzer OLD_VERSION: '0' NEW_VERSION: main LANGUAGE: TypeScript DIAGNOSTIC_OUTPUT: true + REPO_IS_TSGO: true diff --git a/azure-pipelines-gitTests-template.yml b/azure-pipelines-gitTests-template.yml index 4709f11..a405b0f 100644 --- a/azure-pipelines-gitTests-template.yml +++ b/azure-pipelines-gitTests-template.yml @@ -37,6 +37,10 @@ parameters: displayName: Pseudo-random number generator seed type: string default: 'n/a' + - name: REPO_IS_TSGO + displayName: Whether the repo is `typescript-go` + type: boolean + default: false jobs: - job: ListRepos @@ -100,7 +104,7 @@ jobs: corepack enable corepack enable npm mkdir 'RepoResults$(System.JobPositionInPhase)' - node dist/checkGithubRepos ${{ parameters.ENTRYPOINT }} ${{ parameters.OLD_VERSION }} ${{ parameters.NEW_VERSION }} '$(Pipeline.Workspace)/RepoList/repos.json' $(System.TotalJobsInPhase) $(System.JobPositionInPhase) 'RepoResults$(System.JobPositionInPhase)' ${{ parameters.DIAGNOSTIC_OUTPUT }} ${{ parameters.PRNG_SEED }} + node dist/checkReposScheduled ${{ parameters.ENTRYPOINT }} ${{ parameters.OLD_VERSION }} ${{ parameters.NEW_VERSION }} '$(Pipeline.Workspace)/RepoList/repos.json' $(System.TotalJobsInPhase) $(System.JobPositionInPhase) 'RepoResults$(System.JobPositionInPhase)' ${{ parameters.DIAGNOSTIC_OUTPUT }} ${{ parameters.PRNG_SEED }} displayName: 'Run TypeScript on repos' continueOnError: true - publish: 'RepoResults$(System.JobPositionInPhase)' @@ -127,7 +131,7 @@ jobs: - script: | npm ci npm run build - node dist/postGithubIssue ${{ parameters.ENTRYPOINT }} ${{ parameters.LANGUAGE }} ${{ parameters.REPO_COUNT }} ${{ parameters.REPO_START_INDEX }} '$(Pipeline.Workspace)' '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)' '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)&view=artifacts&type=publishedArtifacts' ${{ parameters.POST_RESULT }} '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_apis/build/builds/$(Build.BuildId)/artifacts' + node dist/postGithubIssue ${{ parameters.ENTRYPOINT }} ${{ parameters.LANGUAGE }} ${{ parameters.REPO_COUNT }} ${{ parameters.REPO_START_INDEX }} '$(Pipeline.Workspace)' '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)' '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)&view=artifacts&type=publishedArtifacts' ${{ parameters.POST_RESULT }} '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_apis/build/builds/$(Build.BuildId)/artifacts' ${{ parameters.REPO_IS_TSGO }} displayName: 'Create issue from new errors' env: GITHUB_PAT: $(typescript-bot-github-PAT-error-deltas) diff --git a/azure-pipelines-userTests.yml b/azure-pipelines-userTests.yml index 95e9c56..a913980 100644 --- a/azure-pipelines-userTests.yml +++ b/azure-pipelines-userTests.yml @@ -135,7 +135,7 @@ jobs: corepack enable corepack enable npm mkdir 'RepoResults$(System.JobPositionInPhase)' - node dist/checkUserTestRepos ${{ parameters.ENTRYPOINT }} ${{ parameters.OLD_TS_REPO_URL }} ${{ parameters.OLD_HEAD_REF }} ${{ parameters.SOURCE_ISSUE }} ${{ parameters.TOP_REPOS }} '$(Pipeline.Workspace)/RepoList/repos.json' $(System.TotalJobsInPhase) $(System.JobPositionInPhase) 'RepoResults$(System.JobPositionInPhase)' ${{ parameters.DIAGNOSTIC_OUTPUT }} ${{ parameters.PRNG_SEED }} + node dist/checkReposTriggered ${{ parameters.ENTRYPOINT }} ${{ parameters.OLD_TS_REPO_URL }} ${{ parameters.OLD_HEAD_REF }} ${{ parameters.SOURCE_ISSUE }} ${{ parameters.TOP_REPOS }} '$(Pipeline.Workspace)/RepoList/repos.json' $(System.TotalJobsInPhase) $(System.JobPositionInPhase) 'RepoResults$(System.JobPositionInPhase)' ${{ parameters.DIAGNOSTIC_OUTPUT }} ${{ parameters.PRNG_SEED }} displayName: 'Run user tests' - publish: 'RepoResults$(System.JobPositionInPhase)' artifact: 'RepoResults$(System.JobPositionInPhase)' @@ -161,7 +161,7 @@ jobs: - script: | npm ci npm run build - node dist/postGithubComments ${{ parameters.ENTRYPOINT }} ${{ parameters.REQUESTING_USER }} ${{ parameters.SOURCE_ISSUE }} ${{ parameters.STATUS_COMMENT }} ${{ parameters.DISTINCT_ID }} ${{ parameters.TOP_REPOS }} '$(Pipeline.Workspace)' '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)&view=artifacts&type=publishedArtifacts' ${{ parameters.POST_RESULT }} ${{ parameters.REPO_COUNT }} '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_apis/build/builds/$(Build.BuildId)/artifacts' + node dist/postGithubComments ${{ parameters.ENTRYPOINT }} ${{ parameters.REQUESTING_USER }} ${{ parameters.SOURCE_ISSUE }} ${{ parameters.STATUS_COMMENT }} ${{ parameters.DISTINCT_ID }} ${{ parameters.TOP_REPOS }} '$(Pipeline.Workspace)' '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)&view=artifacts&type=publishedArtifacts' ${{ parameters.POST_RESULT }} ${{ parameters.REPO_COUNT }} '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_apis/build/builds/$(Build.BuildId)/artifacts' ${{ contains(parameters.OLD_TS_REPO_URL, 'typescript-go') }} displayName: 'Update PR comment with new errors' env: GITHUB_PAT: $(typescript-bot-github-PAT-error-deltas) diff --git a/src/checkGithubRepos.ts b/src/checkReposScheduled.ts similarity index 96% rename from src/checkGithubRepos.ts rename to src/checkReposScheduled.ts index acd8ad6..f419c54 100644 --- a/src/checkGithubRepos.ts +++ b/src/checkReposScheduled.ts @@ -11,7 +11,7 @@ if (argv.length < 11) { const [,, entrypoint, oldTsNpmVersion, newTsNpmVersion, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput, prngSeed, tmpfs] = argv; mainAsync({ - testType: "github", + testType: "scheduled", tmpfs: tmpfs && tmpfs.toLowerCase() === "false" ? false : true, entrypoint: entrypoint as TsEntrypoint, diagnosticOutput: diagnosticOutput.toLowerCase() === "true", @@ -23,6 +23,7 @@ mainAsync({ newTsNpmVersion, resultDirName, prngSeed: prngSeed.toLowerCase() === "n/a" ? undefined : prngSeed, + isGo: true, }).catch(err => { reportError(err, "Unhandled exception"); process.exit(1); diff --git a/src/checkUserTestRepos.ts b/src/checkReposTriggered.ts similarity index 76% rename from src/checkUserTestRepos.ts rename to src/checkReposTriggered.ts index 4cae965..e635c70 100644 --- a/src/checkUserTestRepos.ts +++ b/src/checkReposTriggered.ts @@ -3,16 +3,16 @@ import { mainAsync, reportError, TsEntrypoint } from "./main"; const { argv } = process; -if (argv.length !== 13) { - console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); +if (argv.length < 13) { + console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} ?`); process.exit(-1); } -const [,, entrypoint, oldTsRepoUrl, oldHeadRef, prNumber, buildWithNewWhenOldFails, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput, prngSeed] = argv; +const [,, entrypoint, oldTsRepoUrl, oldHeadRef, prNumber, buildWithNewWhenOldFails, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput, prngSeed, tempfs] = argv; mainAsync({ - testType: "user", - tmpfs: true, + testType: "triggered", + tmpfs: tempfs && tempfs.toLowerCase() === "false" ? false : true, entrypoint: entrypoint as TsEntrypoint, oldTsRepoUrl, oldHeadRef, @@ -24,6 +24,7 @@ mainAsync({ resultDirName, diagnosticOutput: diagnosticOutput.toLowerCase() === "true", prngSeed: prngSeed.toLowerCase() === "n/a" ? undefined : prngSeed, + isGo: oldTsRepoUrl.includes("typescript-go") }).catch(err => { reportError(err, "Unhandled exception"); process.exit(1); diff --git a/src/main.ts b/src/main.ts index 8aba6bc..efca7f7 100644 --- a/src/main.ts +++ b/src/main.ts @@ -56,20 +56,23 @@ interface Params { * Pass undefined to have a seed generated. */ prngSeed: string | undefined; + + /** True if we're testing typescript-go */ + isGo: boolean; } -export interface GitParams extends Params { - testType: "github"; +export interface ScheduledParams extends Params { + testType: "scheduled"; oldTsNpmVersion: string; newTsNpmVersion: string; } -export interface UserParams extends Params { - testType: "user"; +export interface TriggeredParams extends Params { + testType: "triggered"; oldTsRepoUrl: string; oldHeadRef: string; prNumber: number; } -export type TsEntrypoint = "tsc" | "tsserver" | "lsp"; +export type TsEntrypoint = "tsc" | "tsserver" | "fuzzer"; const processCwd = process.cwd(); const packageTimeout = 10 * 60 * 1000; @@ -218,6 +221,7 @@ async function getTsServerRepoResult( rawErrorArtifactPath: string, diagnosticOutput: boolean, isPr: boolean, + isGo: boolean, ): Promise { if (!await cloneRepo(repo, userTestsDir, downloadDir.path, diagnosticOutput)) { @@ -238,15 +242,28 @@ async function getTsServerRepoResult( const rawErrorName = path.basename(rawErrorArtifactPath); const rawErrorPath = path.join(downloadDir.path, rawErrorName); + // Periodically log memory usage of the main fuzzer process + const mainMemoryInterval = diagnosticOutput ? setInterval(async () => { + const rssKb = await getProcessRssKb(process.pid); + if (rssKb !== undefined) { + const rssMb = Math.round(rssKb / 1024); + console.error(`Main process memory (pid ${process.pid}): ${rssMb} MB`); + } + }, 30_000) : undefined; + mainMemoryInterval?.unref(); + const lsStart = performance.now(); try { console.log(`Testing with ${newTsServerPath} (new)`); - const newSpawnResult = await spawnWithTimeoutAsync(repoDir, process.argv[0], [path.join(__dirname, "utils", "exerciseServer.js"), repoDir, replayScriptPath, newTsServerPath, diagnosticOutput.toString(), prng.string(10)], executionTimeout); + + const newSpawnResult = isGo ? + await spawnWithTimeoutAsync(repoDir, process.argv[0], [path.join(__dirname, "utils", "exerciseLspServer.js"), repoDir, replayScriptPath, newTsServerPath, diagnosticOutput.toString(), prng.string(10), "n/a"], executionTimeout) : + await spawnWithTimeoutAsync(repoDir, process.argv[0], [path.join(__dirname, "utils", "exerciseServer.js"), repoDir, replayScriptPath, newTsServerPath, diagnosticOutput.toString(), prng.string(10)], executionTimeout); if (!newSpawnResult) { // CONSIDER: It might be interesting to treat timeouts as failures, but they'd be harder to baseline and more likely to have flaky repros console.log(`New server timed out after ${executionTimeout} ms`); - return { status: "Unknown failure" }; + return { status: "Timeout" }; } if (diagnosticOutput) { @@ -265,8 +282,10 @@ async function getTsServerRepoResult( console.log("No issues found"); break; case exercise.EXIT_LANGUAGE_SERVICE_DISABLED: - console.log("Skipping since language service was disabled"); - return { status: "Language service disabled in new TS" }; + if (!isGo) { + console.log("Skipping since language service was disabled"); + return { status: "Language service disabled in new TS" }; + } case exercise.EXIT_SERVER_CRASH: case exercise.EXIT_SERVER_ERROR: case exercise.EXIT_SERVER_EXIT_FAILED: @@ -287,12 +306,15 @@ async function getTsServerRepoResult( if (newServerFailed) { console.log(`Issue found in ${newTsServerPath} (new):`); - console.log(insetLines(prettyPrintServerHarnessOutput(newSpawnResult.stdout, /*filter*/ false))); - await fs.promises.writeFile(rawErrorPath, prettyPrintServerHarnessOutput(newSpawnResult.stdout, /*filter*/ false)); + const harnessOutput = isGo ? prettyPrintLspHarnessOutput(newSpawnResult.stdout, /*filter*/ false) : prettyPrintServerHarnessOutput(newSpawnResult.stdout, /*filter*/ false); + console.log(insetLines(harnessOutput)); + await fs.promises.writeFile(rawErrorPath, harnessOutput); } console.log(`Testing with ${oldTsServerPath} (old)`); - const oldSpawnResult = await spawnWithTimeoutAsync(repoDir, process.argv[0], [path.join(__dirname, "..", "node_modules", "@typescript", "server-replay", "replay.js"), repoDir, replayScriptPath, oldTsServerPath, "-u"], executionTimeout);; + const oldSpawnResult = isGo ? + await spawnWithTimeoutAsync(repoDir, process.argv[0], [path.join(__dirname, "utils", "replayLspServer.js"), repoDir, replayScriptPath, oldTsServerPath, diagnosticOutput.toString()], executionTimeout) : + await spawnWithTimeoutAsync(repoDir, process.argv[0], [path.join(__dirname, "..", "node_modules", "@typescript", "server-replay", "replay.js"), repoDir, replayScriptPath, oldTsServerPath, "-u"], executionTimeout); if (diagnosticOutput && oldSpawnResult) { console.log("Raw spawn results (old):"); @@ -307,27 +329,40 @@ async function getTsServerRepoResult( } if (oldServerFailed) { + const oldHarnessOutput = oldSpawnResult?.stdout && + (isGo ? + prettyPrintLspHarnessOutput(oldSpawnResult.stdout, /*filter*/ false) : + prettyPrintServerHarnessOutput(oldSpawnResult.stdout, /*filter*/ false)); console.log(`Issue found in ${oldTsServerPath} (old):`); console.log( insetLines( - oldSpawnResult?.stdout - ? prettyPrintServerHarnessOutput(oldSpawnResult.stdout, /*filter*/ false) - : `Timed out after ${executionTimeout} ms`)); + oldHarnessOutput ?? `Timed out after ${executionTimeout} ms`)); await fs.promises.writeFile(rawErrorPath, oldSpawnResult?.stdout ?? `Timed out after ${executionTimeout} ms`); // We don't want to drown PRs with comments. // Override the results to say nothing interesting changed. if (isPr && newServerFailed && oldSpawnResult) { - const oldOut = parseServerHarnessOutput(oldSpawnResult.stdout); - const newOut = parseServerHarnessOutput(newSpawnResult.stdout); - - if ( - typeof oldOut !== "string" && typeof newOut !== "string" - && oldOut.request_seq === newOut.request_seq - && oldOut.command === newOut.command - ) { - return { status: "Detected no interesting changes" }; + if (isGo) { + const oldOut = parseLspHarnessOutput(oldSpawnResult.stdout); + const newOut = parseLspHarnessOutput(newSpawnResult.stdout); + if ( + typeof oldOut !== "string" && typeof newOut !== "string" + && oldOut.seq === newOut.seq + && oldOut.method === newOut.method + ) { + return { status: "Detected no interesting changes" }; + } + } else { + const oldOut = parseServerHarnessOutput(oldSpawnResult.stdout); + const newOut = parseServerHarnessOutput(newSpawnResult.stdout); + if ( + typeof oldOut !== "string" && typeof newOut !== "string" + && oldOut.request_seq === newOut.request_seq + && oldOut.command === newOut.command + ) { + return { status: "Detected no interesting changes" }; + } } } } @@ -464,16 +499,15 @@ export async function getLSPResult( } } -function groupErrors(summaries: Summary[]) { +function groupErrors(summaries: Summary[], isGo: boolean) { const groupedOldErrors = new Map(); const groupedNewErrors = new Map(); let group: Map; let error: ServerHarnessOutput | LspHarnessOutput | string; for (const summary of summaries) { - const isLsp = summary.entrypoint === "lsp"; if (summary.tsServerResult.newServerFailed) { // Group new errors - error = isLsp + error = isGo ? parseLspHarnessOutput(summary.tsServerResult.newSpawnResult.stdout) : parseServerHarnessOutput(summary.tsServerResult.newSpawnResult.stdout); group = groupedNewErrors; @@ -482,7 +516,7 @@ function groupErrors(summaries: Summary[]) { // Group old errors const { oldSpawnResult } = summary.tsServerResult; error = oldSpawnResult?.stdout - ? (isLsp ? parseLspHarnessOutput(oldSpawnResult.stdout) : parseServerHarnessOutput(oldSpawnResult.stdout)) + ? (isGo ? parseLspHarnessOutput(oldSpawnResult.stdout) : parseServerHarnessOutput(oldSpawnResult.stdout)) : `Timed out after ${executionTimeout} ms`; group = groupedOldErrors; @@ -493,7 +527,7 @@ function groupErrors(summaries: Summary[]) { const key = typeof error === "string" ? getHash([error]) - : summary.entrypoint === "lsp" + : isGo ? getHashForGoStack(error.message) : getHashForStack(error.message); const value = group.get(key) ?? []; @@ -504,29 +538,28 @@ function groupErrors(summaries: Summary[]) { return { groupedOldErrors, groupedNewErrors } } -function getErrorMessage(output: string): string { +function getJSErrorMessage(output: string): string { const error = parseServerHarnessOutput(output); return typeof error === "string" ? error : getErrorMessageFromStack(error.message); } -function getErrorMessageForEntrypoint(output: string, entrypoint: TsEntrypoint): string { - return entrypoint === "lsp" ? getLspErrorMessage(output) : getErrorMessage(output); +function getErrorMessage(output: string, isGo: boolean): string { + return isGo ? getLspErrorMessage(output) : getJSErrorMessage(output); } -function prettyPrintForEntrypoint(output: string, filter: boolean, entrypoint: TsEntrypoint): string { - return entrypoint === "lsp" ? prettyPrintLspHarnessOutput(output, filter) : prettyPrintServerHarnessOutput(output, filter); +function prettyPrint(output: string, filter: boolean, isGo: boolean): string { + return isGo ? prettyPrintLspHarnessOutput(output, filter) : prettyPrintServerHarnessOutput(output, filter); } -function createOldErrorSummary(summaries: Summary[]): string { +function createOldErrorSummary(summaries: Summary[], isGo: boolean): string { const { oldSpawnResult } = summaries[0].tsServerResult; - const entrypoint = summaries[0].entrypoint; const oldServerError = oldSpawnResult?.stdout - ? prettyPrintForEntrypoint(oldSpawnResult.stdout, /*filter*/ true, entrypoint) + ? prettyPrint(oldSpawnResult.stdout, /*filter*/ true, isGo) : `Timed out after ${executionTimeout} ms`; - const errorMessage = oldSpawnResult?.stdout ? getErrorMessageForEntrypoint(oldSpawnResult.stdout, entrypoint) : oldServerError; + const errorMessage = oldSpawnResult?.stdout ? getErrorMessage(oldSpawnResult.stdout, isGo) : oldServerError; let text = `
@@ -607,14 +640,13 @@ npx tsreplay ./${summary.repo.name} ./${summary.replayScriptName} { - const entrypoint = summaries[0].entrypoint; +async function createNewErrorSummaryAsync(summaries: Summary[], isGo: boolean): Promise { const stdout = summaries[0].tsServerResult.newSpawnResult.stdout; - let text = `

${getErrorMessageForEntrypoint(stdout, entrypoint)}

+ let text = `

${getErrorMessage(stdout, isGo)}

\`\`\` -${prettyPrintForEntrypoint(stdout, /*filter*/ true, entrypoint)} +${prettyPrint(stdout, /*filter*/ true, isGo)} \`\`\`

Affected repos

`; @@ -628,7 +660,31 @@ ${prettyPrintForEntrypoint(stdout, /*filter*/ true, entrypoint)} ${owner + mdEscape(summary.repo.name)} Raw error text: ${summary.rawErrorArtifactPath} in the artifact folder
Replay commands: ${summary.replayScriptArtifactPath} in the artifact folder -

Last few requests

+`; + + // Show what happened with the old server + const { oldServerFailed, oldSpawnResult } = summary.tsServerResult; + if (!oldServerFailed) { + text += `

Old server result

+

The old server completed successfully for this repo.

+`; + } + else if (!oldSpawnResult) { + text += `

Old server result

+

The old server timed out after ${executionTimeout} ms.

+`; + } + else { + const oldHarnessOutput = prettyPrint(oldSpawnResult.stdout, /*filter*/ true, isGo); + text += `

Old server result

+ +\`\`\` +${oldHarnessOutput} +\`\`\` +`; + } + + text += `

Last few requests

\`\`\`json ${summary.replayScript} @@ -926,7 +982,7 @@ function getWorkerRepos(allRepos: readonly git.Repo[], workerCount: number, work return allRepos.slice(start, end); } -export async function mainAsync(params: GitParams | UserParams): Promise { +export async function mainAsync(params: ScheduledParams | TriggeredParams): Promise { if (params.prngSeed) { prng.seed(params.prngSeed); } @@ -958,7 +1014,7 @@ export async function mainAsync(params: GitParams | UserParams): Promise { // An object is easier to de/serialize than a real map const statusCounts: { [P in RepoStatus]?: number } = {}; - const isPr = params.testType === "user" && !!params.prNumber + const isPr = params.testType === "triggered" && !!params.prNumber var summaries: Summary[] = []; @@ -986,9 +1042,9 @@ export async function mainAsync(params: GitParams | UserParams): Promise { repoResult = await getTscRepoResult(repo, userTestsDir, oldTsEntrypointPath!, newTsEntrypointPath, params.buildWithNewWhenOldFails, downloadDir, diagnosticOutput); break; case "tsserver": - repoResult = await getTsServerRepoResult(repo, userTestsDir, oldTsEntrypointPath!, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput, isPr); + repoResult = await getTsServerRepoResult(repo, userTestsDir, oldTsEntrypointPath!, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput, isPr, params.isGo); break; - case "lsp": + case "fuzzer": repoResult = await getLSPResult(repo, userTestsDir, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput); break; default: @@ -1049,17 +1105,17 @@ export async function mainAsync(params: GitParams | UserParams): Promise { // Group errors and create summary files. if (summaries.length > 0) { - const { groupedOldErrors, groupedNewErrors } = groupErrors(summaries); + const { groupedOldErrors, groupedNewErrors } = groupErrors(summaries, params.isGo); for (let [key, value] of groupedOldErrors) { - const summary = createOldErrorSummary(value); + const summary = createOldErrorSummary(value, params.isGo); const resultFileName = `!${key}.${resultFileNameSuffix}`; // Exclamation point makes the file to be put first when ordering. await fs.promises.writeFile(path.join(resultDirPath, resultFileName), summary, { encoding: "utf-8" }); } for (let [key, value] of groupedNewErrors) { - const summary = await createNewErrorSummaryAsync(value); + const summary = await createNewErrorSummaryAsync(value, params.isGo); const resultFileName = `${key}.${resultFileNameSuffix}`; await fs.promises.writeFile(path.join(resultDirPath, resultFileName), summary, { encoding: "utf-8" }); @@ -1080,7 +1136,7 @@ export async function mainAsync(params: GitParams | UserParams): Promise { newTsResolvedVersion: newTsResolvedVersion, oldTsResolvedVersion: oldTsResolvedVersion || "", statusCounts, - lspRequestStats: params.entrypoint === "lsp" ? aggregateLspStats : undefined, + lspRequestStats: params.entrypoint === "fuzzer" ? aggregateLspStats : undefined, }; await fs.promises.writeFile(path.join(resultDirPath, metadataFileName), JSON.stringify(metadata), { encoding: "utf-8" }); } @@ -1224,6 +1280,7 @@ function filterToTsserverLines(stackLines: string): string { export interface LspHarnessOutput { method: string; message: string; + seq: number; } function parseLspHarnessOutput(output: string): LspHarnessOutput | string { @@ -1288,16 +1345,16 @@ function makeMarkdownLink(url: string) { : `[${mdEscape(match[1])}](${url})`; } -async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Promise<{ oldTsEntrypointPath: string | undefined, oldTsResolvedVersion: string | undefined, newTsEntrypointPath: string, newTsResolvedVersion: string }> { +async function downloadTsAsync(cwd: string, params: ScheduledParams | TriggeredParams): Promise<{ oldTsEntrypointPath: string | undefined, oldTsResolvedVersion: string | undefined, newTsEntrypointPath: string, newTsResolvedVersion: string }> { const entrypoint = params.entrypoint; - if (params.testType === "user") { + if (params.testType === "triggered") { console.log("running user test, downloading TS from repo"); - if (params.entrypoint === "lsp") { + if (params.entrypoint === "fuzzer") { throw new Error("Not implemented"); } - const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = await downloadTsRepoAsync(cwd, params.oldTsRepoUrl, params.oldHeadRef, entrypoint); + const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = await downloadTsRepoAsync(cwd, params.oldTsRepoUrl, params.oldHeadRef, entrypoint, params.isGo); // We need to handle the ref/pull/*/merge differently as it is not a branch and cannot be pulled during clone. - const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = await downloadTsPrAsync(cwd, params.oldTsRepoUrl, params.prNumber, entrypoint); + const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = await downloadTsPrAsync(cwd, params.oldTsRepoUrl, params.prNumber, entrypoint, params.isGo); return { oldTsEntrypointPath, @@ -1306,15 +1363,17 @@ async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Pro newTsResolvedVersion }; } - else if (params.testType === "github") { - const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = params.entrypoint === "lsp" ? + else if (params.testType === "scheduled") { + const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = params.entrypoint === "fuzzer" ? { tsEntrypointPath: undefined, resolvedVersion: undefined } : await downloadTsNpmAsync(cwd, params.oldTsNpmVersion, entrypoint); - const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = params.entrypoint === "lsp" ? + const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = params.entrypoint === "fuzzer" ? params.newTsNpmVersion === "main" ? - await downloadTsRepoAsync(cwd, "https://github.com/microsoft/typescript-go.git", /*headRef*/ "main", entrypoint) : + await downloadTsRepoAsync(cwd, "https://github.com/microsoft/typescript-go.git", /*headRef*/ "main", entrypoint, params.isGo) : await downloadTsNativePreviewNpmAsync(cwd, params.newTsNpmVersion) : - await downloadTsNpmAsync(cwd, params.newTsNpmVersion, entrypoint); + params.isGo ? + await downloadTsNativePreviewNpmAsync(cwd, params.newTsNpmVersion) : + await downloadTsNpmAsync(cwd, params.newTsNpmVersion, entrypoint); return { oldTsEntrypointPath, @@ -1328,9 +1387,9 @@ async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Pro } } -export async function downloadTsRepoAsync(cwd: string, repoUrl: string, headRef: string, target: TsEntrypoint): Promise<{ tsEntrypointPath: string, resolvedVersion: string }> { +export async function downloadTsRepoAsync(cwd: string, repoUrl: string, headRef: string, target: TsEntrypoint, isGo: boolean): Promise<{ tsEntrypointPath: string, resolvedVersion: string }> { console.log(`Cloning ${repoUrl} at ref ${headRef}`); - const repoName = repoUrl.includes("typescript-go") ? `typescript-go-${headRef}` : `typescript-${headRef}`; + const repoName = isGo ? `typescript-go-${headRef}` : `typescript-${headRef}`; await git.cloneRepoIfNecessary(cwd, { name: repoName, url: repoUrl, branch: headRef }); const repoPath = path.join(cwd, repoName); @@ -1341,10 +1400,10 @@ export async function downloadTsRepoAsync(cwd: string, repoUrl: string, headRef: }; } -async function downloadTsPrAsync(cwd: string, repoUrl: string, prNumber: number, target: TsEntrypoint): Promise<{ tsEntrypointPath: string, resolvedVersion: string }> { +async function downloadTsPrAsync(cwd: string, repoUrl: string, prNumber: number, target: TsEntrypoint, isGo: boolean): Promise<{ tsEntrypointPath: string, resolvedVersion: string }> { console.log(`Cloning ${repoUrl} at pull ${prNumber}`); - const repoName = repoUrl.includes("typescript-go") ? `typescript-go-${prNumber}` : `typescript-${prNumber}`; + const repoName = isGo ? `typescript-go-${prNumber}` : `typescript-${prNumber}`; console.log(`Building in ${repoName}`); await git.cloneRepoIfNecessary(cwd, { name: repoName, url: repoUrl }); @@ -1365,10 +1424,7 @@ async function buildTs(repoPath: string, entrypoint: TsEntrypoint) { console.log(`Building in ${repoPath}`); if (repoPath.includes("typescript-go")) { - if (entrypoint !== "tsc" && entrypoint !== "lsp") { - throw new Error(`TsEntrypoint '${entrypoint}' is not supported for typescript-go repositories.`); - } - await execAsync(repoPath, `npx hereby build ${entrypoint === "lsp" ? "--assert" : ""}`); + await execAsync(repoPath, `npx hereby build`); return path.join(repoPath, "built", "local", "tsgo"); } else { diff --git a/src/postGithubComments.ts b/src/postGithubComments.ts index c42d481..bbecb81 100644 --- a/src/postGithubComments.ts +++ b/src/postGithubComments.ts @@ -7,15 +7,15 @@ import { asMarkdownInlineCode } from "./utils/markdownUtils"; const { argv } = process; -if (argv.length !== 13) { - console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); +if (argv.length !== 14) { + console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); process.exit(-1); } -const [, , entrypoint, userToTag, prNumber, commentNumber, distinctId, isTop, resultDirPath, artifactsUri, post, repoCount, getArtifactsApi] = argv; +const [, , entrypoint, userToTag, prNumber, commentNumber, distinctId, isTop, resultDirPath, artifactsUri, post, repoCount, getArtifactsApi, isGoStr] = argv; const isTopReposRun = isTop.toLowerCase() === "true"; const postResult = post.toLowerCase() === "true"; - +const isGoRepo = isGoStr.toLowerCase() === "true"; const metadataFilePaths = pu.glob(resultDirPath, `**/${metadataFileName}`); let newTscResolvedVersion: string | undefined; @@ -82,7 +82,7 @@ let header = `@${userToTag} Here are the results of running the ${suiteDescripti ${summary.join("\n")}`; if (!outputs.length) { - git.createComment(entrypoint as TsEntrypoint, +prNumber, +commentNumber, distinctId, postResult, [header], somethingChanged); + git.createComment(isGoRepo, +prNumber, +commentNumber, distinctId, postResult, [header], somethingChanged); } else { const oldErrorHeader = `

:warning: Old server errors :warning:

`; @@ -137,5 +137,5 @@ else { console.log(`Chunk of size ${chunk.length}`); } - git.createComment(entrypoint as TsEntrypoint, +prNumber, +commentNumber, distinctId, postResult, bodyChunks, somethingChanged); + git.createComment(isGoRepo, +prNumber, +commentNumber, distinctId, postResult, bodyChunks, somethingChanged); } diff --git a/src/postGithubIssue.ts b/src/postGithubIssue.ts index c67305c..aa9040d 100644 --- a/src/postGithubIssue.ts +++ b/src/postGithubIssue.ts @@ -6,14 +6,15 @@ import pu = require("./utils/packageUtils"); const { argv } = process; -if (argv.length !== 11) { - console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); +if (argv.length !== 12) { + console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); process.exit(-1); } -const [, , ep, language, repoCount, repoStartIndex, resultDirPath, logUri, artifactsUri, post, getArtifactsApi] = argv; +const [, , ep, language, repoCount, repoStartIndex, resultDirPath, logUri, artifactsUri, post, getArtifactsApi, isGoStr] = argv; const postResult = post.toLowerCase() === "true"; const entrypoint = ep as TsEntrypoint; +const isGoRepo = isGoStr.toLowerCase() === "true"; const metadataFilePaths = pu.glob(resultDirPath, `**/${metadataFileName}`); @@ -45,14 +46,14 @@ for (const path of metadataFilePaths) { } -const title = `${entrypoint === "tsserver" || entrypoint === "lsp" ? `[ServerErrors][${language}]` : `[NewErrors]`} ${newTscResolvedVersion} vs ${oldTscResolvedVersion}`; +const title = `${entrypoint === "tsserver" || entrypoint === "fuzzer" ? `[ServerErrors][${language}]` : `[NewErrors]`} ${newTscResolvedVersion} vs ${oldTscResolvedVersion}`; const description = entrypoint === "tsserver" ? `The following errors were reported by ${newTscResolvedVersion} vs ${oldTscResolvedVersion}` - : entrypoint == "lsp" + : entrypoint == "fuzzer" ? `The following errors were reported by ${newTscResolvedVersion}` : `The following errors were reported by ${newTscResolvedVersion}, but not by ${oldTscResolvedVersion}`; -const pipelineUri = entrypoint === "lsp" ? +const pipelineUri = entrypoint === "fuzzer" ? "https://dev.azure.com/typescript/TypeScript/_build?definitionId=75" : "https://typescript.visualstudio.com/TypeScript/_build?definitionId=48"; let header = `${description} @@ -80,7 +81,7 @@ const outputs = resultPaths.map(p => // tsserver groups results by error, causing the summary to not make sense. Remove the list for now. // See issue: https://github.com/microsoft/typescript-error-deltas/issues/114 -if (entrypoint !== "tsserver" && entrypoint !== "lsp") { +if (entrypoint !== "tsserver" && entrypoint !== "fuzzer") { header += ` ## Investigation Status | Repo | Errors | Outcome | @@ -108,4 +109,4 @@ if (entrypoint !== "tsserver" && entrypoint !== "lsp") { const bodyChunks = [header, ...outputs]; -git.createIssue(entrypoint, postResult, title, bodyChunks, /*sawNewErrors*/ !!outputs.length); \ No newline at end of file +git.createIssue(isGoRepo, postResult, title, bodyChunks, /*sawNewErrors*/ !!outputs.length); \ No newline at end of file diff --git a/src/utils/exerciseLspServer.ts b/src/utils/exerciseLspServer.ts index 8e57607..7cc59ed 100644 --- a/src/utils/exerciseLspServer.ts +++ b/src/utils/exerciseLspServer.ts @@ -4,11 +4,11 @@ import path from "path"; import { performance } from "perf_hooks"; import process from "process"; import randomSeed from "random-seed"; -import { pathToFileURL } from "url"; import * as protocol from "vscode-languageserver-protocol"; import { EXIT_BAD_ARGS, EXIT_SERVER_COMMUNICATION_ERROR, EXIT_SERVER_CRASH, EXIT_SERVER_ERROR, EXIT_UNHANDLED_EXCEPTION } from "./exerciseServerConstants"; import { getProcessRssKb } from "./execUtils"; import * as lsp from "./lspHarness"; +import { getPanicMessageFromStderr } from "./hashStackTrace"; const testDirUriPlaceholder = "@PROJECT_ROOT_URI@"; const testDirPlaceholder = "@PROJECT_ROOT@"; @@ -49,7 +49,9 @@ export async function exerciseLspServer(testDir: string, replayScriptPath: strin } finally { await replayScriptHandle.close(); - await fs.promises.writeFile(statsOutputPath, JSON.stringify(requestStats), { encoding: "utf-8" }); + if (statsOutputPath != "n/a") { + await fs.promises.writeFile(statsOutputPath, JSON.stringify(requestStats), { encoding: "utf-8" }); + } process.chdir(oldCwd); @@ -63,10 +65,6 @@ export async function exerciseLspServer(testDir: string, replayScriptPath: strin } } -function filePathToUri(filePath: string): string { - return pathToFileURL(filePath).toString(); -} - function getLanguageId(filePath: string): string { const ext = path.extname(filePath).toLowerCase(); switch (ext) { @@ -86,6 +84,7 @@ function getLanguageId(filePath: string): string { } async function exerciseLspServerWorker(testDir: string, lspServerPath: string, replayScriptHandle: fs.promises.FileHandle, requestTimes: Record, requestCounts: Record, requestStats: LspRequestStats): Promise { + let seq = 0; const files = await glob.glob("**/*.@(ts|tsx|mts|cts|js|jsx|mjs|cjs)", { cwd: testDir, absolute: true, ignore: ["**/node_modules/**", "**/*.min.js"], nodir: true, follow: false }); const serverArgs: string[] = ["--lsp", "--stdio"]; @@ -105,7 +104,17 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r const server = lsp.startServer(lspServerPath, { args: serverArgs, - }, { traceOutput: diagnosticOutput }); + }); + + // Collect stderr output from the server process + const stderrChunks: string[] = []; + server.stderr.on("data", (chunk: Buffer) => { + const text = chunk.toString(); + stderrChunks.push(text); + if (diagnosticOutput) { + process.stderr.write(text); + } + }); // Periodically log memory usage of the LSP server process and the harness process const memoryLogInterval = diagnosticOutput ? setInterval(async () => { @@ -130,37 +139,45 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r let lastErrorLogMessage = ""; server.handleAnyNotification(async (...args: any[]) => { - console.error("Server sent notification:", ...args); const [method, params] = args; if (method === "window/logMessage" && params?.type === 1) { lastErrorLogMessage = params.message; } + if (method !== "window/logMessage") { + console.error("Server sent notification:", ...args); + } }); let exitExpected = false; server.onError(async ([error, message, count]) => { console.error(`Server connection error: ${error} ${message} ${count}`); - exitExpected = true; - await server.kill(); + await killServer(); process.exit(EXIT_SERVER_COMMUNICATION_ERROR); }); server.onClose((e) => { if (!exitExpected) { - const errorMessage = lastErrorLogMessage || `Server connection closed prematurely: ${e}`; - console.log(JSON.stringify({ method: "unknown", message: errorMessage })); - console.error("Server connection closed prematurely:", e); + const stderrOutput = stderrChunks.join(""); + const panicMsg = getPanicMessageFromStderr(stderrOutput); + const errorMessage = panicMsg || lastErrorLogMessage || `Server connection closed prematurely: ${e}`; + console.log(JSON.stringify({ method: "unknown", message: errorMessage, seq })); + console.error(errorMessage); process.exit(EXIT_SERVER_CRASH); } }); + + async function killServer() { + exitExpected = true; + await server.kill(); + } let documentVersion = 0; - const testDirUrl = filePathToUri(testDir); + const testDirUrl = lsp.filePathToUri(testDir); // Initialize the server const initializeParams: protocol.InitializeParams = { - processId: process.pid, + processId: null, capabilities: { textDocument: { completion: { @@ -264,7 +281,7 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r for (const openFileAbsolutePath of files) { if (prng.random() > skipFileProb) continue; - const openFileUri = filePathToUri(openFileAbsolutePath); + const openFileUri = lsp.filePathToUri(openFileAbsolutePath); if (openFileUris.length === 5) { const closedFileUri = openFileUris.shift()!; @@ -708,14 +725,14 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r console.error("\nShutting down server"); exitExpected = true; // Send shutdown request and exit notification - void request(protocol.ShutdownRequest.method, undefined); - void notify("exit", undefined); + await request(protocol.ShutdownRequest.method, undefined); + await notify("exit", undefined); } catch (e) { console.error("Killing server after unhandled exception"); console.error(e); - exitExpected = true; - await server.kill(); + await killServer(); + clearInterval(memoryLogInterval) process.exit(EXIT_UNHANDLED_EXCEPTION); } @@ -728,6 +745,7 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r params: lsp.RequestToParams[K], prob = 1, ): Promise { + seq++; if (prng.random() > prob) return undefined as any; const replayEntry = { kind: "request", method, params }; @@ -755,9 +773,10 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r else { console.error(errorMessage); } - console.log(JSON.stringify({ method, message: errorMessage })); + console.log(JSON.stringify({ method, message: errorMessage, seq })); - void server.kill(); + await killServer(); + clearInterval(memoryLogInterval); process.exit(EXIT_SERVER_ERROR); } } @@ -766,6 +785,7 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r method: K, params: lsp.NotificationToParams[K], ): Promise { + seq++; const replayEntry = { kind: "notification", method, params }; const replayStr = JSON.stringify(replayEntry).replaceAll(testDirUrl, testDirUriPlaceholder).replaceAll(testDir, testDirPlaceholder); await replayScriptHandle.write(replayStr + "\n"); @@ -781,9 +801,10 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r else { console.error(errorMessage); } - console.log(JSON.stringify({ method, message: errorMessage })); + console.log(JSON.stringify({ method, message: errorMessage, seq })); - await server.kill(); + await killServer(); + clearInterval(memoryLogInterval); process.exit(EXIT_SERVER_ERROR); } } diff --git a/src/utils/gitUtils.ts b/src/utils/gitUtils.ts index f307e8c..682a0e5 100644 --- a/src/utils/gitUtils.ts +++ b/src/utils/gitUtils.ts @@ -16,20 +16,17 @@ export interface Repo { branch?: string; } -function getRepoProperties(entrypoint: TsEntrypoint) { - switch (entrypoint) { - case "tsserver": - case "tsc": - return { - owner: "microsoft", - repo: "typescript", - }; - case "lsp": - return { - owner: "microsoft", - repo: "typescript-go", - }; +function getRepoProperties(isGoRepo: boolean) { + if (isGoRepo) { + return { + owner: "microsoft", + repo: "typescript-go", + }; } + return { + owner: "microsoft", + repo: "typescript", + }; } export async function getPopularRepos(language = "TypeScript", count = 100, repoStartIndex = 0, skipRepos?: string[], cachePath?: string): Promise { @@ -117,8 +114,8 @@ type Result = { export type GitResult = Result & { kind: 'git', title: string } export type UserResult = Result & { kind: 'user', issue_number: number } -export async function createIssue(entrypoint: TsEntrypoint, postResult: boolean, title: string, bodyChunks: readonly string[], sawNewErrors: boolean): Promise { - const repoProperties = getRepoProperties(entrypoint); +export async function createIssue(isGoRepo: boolean, postResult: boolean, title: string, bodyChunks: readonly string[], sawNewErrors: boolean): Promise { + const repoProperties = getRepoProperties(isGoRepo); const issue = { ...repoProperties, title, @@ -171,8 +168,8 @@ export async function createIssue(entrypoint: TsEntrypoint, postResult: boolean, } } -export async function createComment(entrypoint: TsEntrypoint, prNumber: number, statusComment: number, distinctId: string, postResult: boolean, bodyChunks: readonly string[], somethingChanged: boolean): Promise { - const repoProperties = getRepoProperties(entrypoint); +export async function createComment(isGoRepo: boolean, prNumber: number, statusComment: number, distinctId: string, postResult: boolean, bodyChunks: readonly string[], somethingChanged: boolean): Promise { + const repoProperties = getRepoProperties(isGoRepo); const newComments = bodyChunks.map(body => ({ ...repoProperties, issue_number: prNumber, @@ -213,8 +210,7 @@ export async function createComment(entrypoint: TsEntrypoint, prNumber: number, // Get status comment contents const statusCommentResp = await kit.issues.getComment({ comment_id: statusComment, - owner: "Microsoft", - repo: "TypeScript", + ...repoProperties, }); const oldComment = statusCommentResp.data.body; @@ -231,9 +227,8 @@ export async function createComment(entrypoint: TsEntrypoint, prNumber: number, // Update status comment await kit.issues.updateComment({ comment_id: statusComment, - owner: "Microsoft", - repo: "TypeScript", body: newComment, + ...repoProperties, }); // Repeat; someone may have edited the comment at the same time. diff --git a/src/utils/hashStackTrace.ts b/src/utils/hashStackTrace.ts index 98a5b70..0a30ae9 100644 --- a/src/utils/hashStackTrace.ts +++ b/src/utils/hashStackTrace.ts @@ -50,4 +50,13 @@ export function getErrorMessageFromStack(stack: string): string { const stackLines = stack.split(/\r?\n/, 2); return stackLines[1]; +} + +export function getPanicMessageFromStderr(stderr: string): string | undefined { + const stackLines = stderr.split(/\r?\n/); + const panicLine = stackLines.findIndex(line => line.trim().startsWith("panic:")); + if (panicLine >= 0) { + return stackLines.slice(panicLine).join("\n"); + } + return undefined; } \ No newline at end of file diff --git a/src/utils/lspHarness.ts b/src/utils/lspHarness.ts index f52599c..7ce58a0 100644 --- a/src/utils/lspHarness.ts +++ b/src/utils/lspHarness.ts @@ -1,6 +1,8 @@ import * as cp from "child_process"; import * as rpc from "vscode-jsonrpc/node"; import * as protocol from "vscode-languageserver-protocol"; +import { pathToFileURL } from "url"; +import { Readable } from "stream"; export interface ServerOptions { args?: string[]; @@ -13,6 +15,7 @@ export interface LanguageServer { sendRequest: (method: K, params: RequestToParams[K]) => Promise; sendRequestUntyped: (method: string, params: object) => Promise; sendNotification: (method: K, params: NotificationToParams[K]) => Promise; + sendNotificationUntyped: (method: string, params: unknown) => Promise; handleRequest: (method: K, handler: (params: RequestToParams[K]) => Promise) => void; handleAnyRequest: (handler: (...args: any[]) => Promise) => void; @@ -23,16 +26,17 @@ export interface LanguageServer { onError: rpc.Event<[Error, rpc.Message | undefined, number | undefined]>; onClose: rpc.Event; + stderr: Readable; } -export function startServer(serverPath: string, options: ServerOptions = {}, otherOptions?: { traceOutput?: boolean; }): LanguageServer { +export function startServer(serverPath: string, options: ServerOptions = {}): LanguageServer { const serverProc = cp.spawn(serverPath, options.args ?? [], { env: options.env, // execArgv: options.execArgv, // options.execArgv ?? process.execArgv?.map(arg => bumpDebugPort(arg)), stdio: [ "pipe", // stdin "pipe", // stdout - otherOptions?.traceOutput ? "inherit" : "ignore" // stderr + "pipe", // stderr ], }); @@ -48,6 +52,7 @@ export function startServer(serverPath: string, options: ServerOptions = {}, oth sendRequest, sendRequestUntyped, sendNotification, + sendNotificationUntyped, handleRequest, handleAnyRequest, handleNotification, @@ -55,17 +60,28 @@ export function startServer(serverPath: string, options: ServerOptions = {}, oth kill, onError: connection.onError, onClose: connection.onClose, + stderr: serverProc.stderr, }; function sendRequest(method: K, params: RequestToParams[K]): Promise { - return connection.sendRequest(method, params); + return sendRequestUntyped(method, params) as Promise; } - function sendRequestUntyped(method: string, params: object): Promise { + function sendRequestUntyped(method: string, params: unknown): Promise { + if (params === undefined) { + return connection.sendRequest(method); + } return connection.sendRequest(method, params); } function sendNotification(method: K, params: NotificationToParams[K]): Promise { + return sendNotificationUntyped(method, params); + } + + function sendNotificationUntyped(method: string, params: unknown): Promise { + if (params === undefined) { + return connection.sendNotification(method); + } return connection.sendNotification(method, params); } @@ -203,4 +219,8 @@ export interface NotificationToParams { [protocol.WillSaveTextDocumentNotification.method]: protocol.WillSaveTextDocumentParams; [protocol.DidChangeWatchedFilesNotification.method]: protocol.DidChangeWatchedFilesParams; [protocol.PublishDiagnosticsNotification.method]: protocol.PublishDiagnosticsParams; +} + +export function filePathToUri(filePath: string): string { + return pathToFileURL(filePath).toString(); } \ No newline at end of file diff --git a/src/utils/replayLspServer.ts b/src/utils/replayLspServer.ts new file mode 100644 index 0000000..fd52efe --- /dev/null +++ b/src/utils/replayLspServer.ts @@ -0,0 +1,221 @@ +import readline from "readline"; +import fs from "fs"; +import * as lsp from "./lspHarness"; +import { getProcessRssKb } from "./execUtils"; +import { EXIT_BAD_ARGS, EXIT_SERVER_COMMUNICATION_ERROR, EXIT_SERVER_CRASH, EXIT_SERVER_ERROR, EXIT_UNHANDLED_EXCEPTION } from "./exerciseServerConstants"; +import path from "path"; +import events from "node:events"; +import { ShutdownRequest } from "vscode-languageserver-protocol"; +import { getPanicMessageFromStderr } from "./hashStackTrace"; + + +const argv = process.argv; +if (argv.length !== 6) { + console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); + process.exit(EXIT_BAD_ARGS); +} + +const [testDir, replayPath, lspServerPath, diag] = argv.slice(2); +const diagnosticOutput = diag.toLocaleLowerCase() === "true"; + +replayServer(testDir, lspServerPath, replayPath).catch(e => { + console.error(e); + process.exit(EXIT_UNHANDLED_EXCEPTION); +}); + +async function replayServer(testDir: string, lspServerPath: string, replayPath: string) { + const testDirUri = lsp.filePathToUri(testDir); + const rl = readline.createInterface({ + input: fs.createReadStream(replayPath), + crlfDelay: Infinity, + }); + + let rootDirPlaceholder = "@PROJECT_ROOT@" + let rootDirUriPlaceHolder = "@PROJECT_ROOT_URI@" + let serverArgs = ["--lsp", "--stdio"]; + + let isFirstLine = true; + let messages: Message[] = []; + + rl.on("line", (line) => { + try { + // Ignore blank lines + if (line.trim().length === 0) { + return; + } + + if (isFirstLine) { + const obj = JSON.parse(line); + if (!obj.command) { + rootDirPlaceholder = obj.rootDirPlaceholder || rootDirPlaceholder; + rootDirUriPlaceHolder = obj.rootDirUriPlaceHolder || rootDirUriPlaceHolder; + serverArgs = obj.serverArgs || serverArgs; + return; + } + } + + const message = JSON.parse(line.replace(new RegExp(rootDirPlaceholder, "g"), testDir).replace(new RegExp(rootDirUriPlaceHolder, "g"), testDirUri)); + messages.push(message); + } catch (e) { + console.log(e); + } finally { + isFirstLine = false; + } + }); + await events.once(rl, 'close'); + + await replayServerWorker(testDir, lspServerPath, messages, serverArgs); + + console.log("Replay completed successfully"); +} + +type Message = { + kind: "request" | "notification"; + method: string; + params?: any; +} + +async function replayServerWorker(testDir: string, lspServerPath: string, messages: Message[], serverArgs: string[]) { + let seq = 0; + const server = lsp.startServer(lspServerPath, { + args: serverArgs, + }); + + // Collect stderr output from the server process + const stderrChunks: string[] = []; + server.stderr.on("data", (chunk: Buffer) => { + const text = chunk.toString(); + stderrChunks.push(text); + if (diagnosticOutput) { + process.stderr.write(text); + } + }); + + // Periodically log memory usage of the LSP server process and the harness process + const memoryLogInterval = diagnosticOutput ? setInterval(async () => { + const serverRssKb = await getProcessRssKb(server.pid); + if (serverRssKb !== undefined) { + const rssMb = Math.round(serverRssKb / 1024); + console.error(`LSP server memory (pid ${server.pid}): ${rssMb} MB`); + } + const harnessRssKb = await getProcessRssKb(process.pid); + if (harnessRssKb !== undefined) { + const rssMb = Math.round(harnessRssKb / 1024); + console.error(`Harness memory (pid ${process.pid}): ${rssMb} MB`); + } + }, 30_000) : undefined; + memoryLogInterval?.unref(); + + server.handleAnyRequest(async (...args) => { + console.error("Server sent request:", ...args); + }); + + // Capture the last error-level log message from the server (e.g. Go panic stack traces) + let lastErrorLogMessage = ""; + + server.handleAnyNotification(async (...args: any[]) => { + console.error("Server sent notification:", ...args); + const [method, params] = args; + if (method === "window/logMessage" && params?.type === 1) { + lastErrorLogMessage = params.message; + } + if (method !== "window/logMessage") { + console.error("Server sent notification:", ...args); + } + }); + + let exitExpected = false; + server.onError(async ([error, message, count]) => { + console.error(`Server connection error: ${error} ${message} ${count}`); + await killServer(); + process.exit(EXIT_SERVER_COMMUNICATION_ERROR); + }); + + server.onClose((e) => { + if (!exitExpected) { + const stderrOutput = stderrChunks.join(""); + const panicMsg = getPanicMessageFromStderr(stderrOutput); + const errorMessage = panicMsg || lastErrorLogMessage || `Server connection closed prematurely: ${e}`; + console.log(JSON.stringify({ method: "unknown", message: errorMessage, seq })); + console.error(errorMessage); + process.exit(EXIT_SERVER_CRASH); + } + }); + + async function killServer() { + exitExpected = true; + await server.kill(); + } + + try { + for (const msg of messages) { + if (msg.kind === "request") { + if (msg.method === ShutdownRequest.method) { + exitExpected = true; + } + await request(msg.method, msg.params); + } + else if (msg.kind === "notification") { + await notify(msg.method, msg.params); + } + } + } catch (e: any) { + console.error("Killing server after unhandled exception"); + console.error(e); + + await killServer(); + clearInterval(memoryLogInterval) + process.exit(EXIT_UNHANDLED_EXCEPTION); + } + + await killServer(); + + clearInterval(memoryLogInterval); + + async function request( + method: string, + params: any + ): Promise { + seq++; + try { + return await server.sendRequestUntyped(method, params); + } catch (e: any) { + const errorMessage = lastErrorLogMessage || e.message || "Unknown error"; + if (diagnosticOutput) { + console.error(`Request failed:\n${JSON.stringify({ method, params }, undefined, 2)}\n${e}`); + } + else { + console.error(errorMessage); + } + console.log(JSON.stringify({ method, message: errorMessage, seq })); + + await killServer(); + clearInterval(memoryLogInterval); + process.exit(EXIT_SERVER_ERROR); + } + } + + async function notify( + method: string, + params: any, + ): Promise { + seq++; + try { + await server.sendNotificationUntyped(method, params); + } + catch (e: any) { + const errorMessage = lastErrorLogMessage || e.message || "Unknown error"; + if (diagnosticOutput) { + console.error(`Notification failed:\n${JSON.stringify({ method, params }, undefined, 2)}\n${e}`); + } + else { + console.error(errorMessage); + } + console.log(JSON.stringify({ method, message: errorMessage, seq })); + + await killServer(); + clearInterval(memoryLogInterval); + process.exit(EXIT_SERVER_ERROR); + } + } +} \ No newline at end of file diff --git a/test/__snapshots__/main.test.ts.snap b/test/__snapshots__/main.test.ts.snap index 6966dc4..db7cf18 100644 --- a/test/__snapshots__/main.test.ts.snap +++ b/test/__snapshots__/main.test.ts.snap @@ -36,6 +36,16 @@ Req #123 - cursedCommand MockRepoOwner/MockRepoName Raw error text: RepoResults123/MockRepoOwner.MockRepoName.rawError.txt in the artifact folder
Replay commands: RepoResults123/MockRepoOwner.MockRepoName.replay.txt in the artifact folder +

Old server result

+ +\`\`\` +Req #123 - cursedCommand + at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1) + at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2) + at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3) + at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4) + at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5) +\`\`\`

Last few requests

\`\`\`json @@ -222,6 +232,16 @@ Req #123 - cursedCommand MockRepoOwner/MockRepoName Raw error text: RepoResults123/MockRepoOwner.MockRepoName.rawError.txt in the artifact folder
Replay commands: RepoResults123/MockRepoOwner.MockRepoName.replay.txt in the artifact folder +

Old server result

+ +\`\`\` +Req #123 - cursedCommand + at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1) + at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2) + at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3) + at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4) + at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5) +\`\`\`

Last few requests

\`\`\`json diff --git a/test/getTscErrors.test.ts b/test/getTscErrors.test.ts index b8d94d9..14217e0 100644 --- a/test/getTscErrors.test.ts +++ b/test/getTscErrors.test.ts @@ -10,7 +10,7 @@ describe("getErrors", () => { if (!existsSync("./testDownloads/getErrors")) { mkdirSync("./testDownloads/getErrors", { recursive: true }); } - await downloadTsRepoAsync('./testDownloads/getErrors', 'https://github.com/sandersn/typescript', 'test-fake-error', 'tsc'); + await downloadTsRepoAsync('./testDownloads/getErrors', 'https://github.com/sandersn/typescript', 'test-fake-error', 'tsc', false); } }); diff --git a/test/main.test.ts b/test/main.test.ts index 51c5cd3..bd0df98 100644 --- a/test/main.test.ts +++ b/test/main.test.ts @@ -139,7 +139,7 @@ describe("main", () => { else if (actualFs.existsSync("./testDownloads/main/typescript-test-fake-error")) { execSync("cd ./testDownloads/main/typescript-test-fake-error && git restore . && cd ..") } - await downloadTsRepoAsync('./testDownloads/main', 'https://github.com/sandersn/typescript', 'test-fake-error', 'tsc') + await downloadTsRepoAsync('./testDownloads/main', 'https://github.com/sandersn/typescript', 'test-fake-error', 'tsc', false) }); it("outputs server errors", async () => { @@ -154,7 +154,7 @@ describe("main", () => { }); await mainAsync({ - testType: "github", + testType: "scheduled", tmpfs: false, entrypoint: 'tsserver', diagnosticOutput: false, @@ -166,6 +166,7 @@ describe("main", () => { newTsNpmVersion: 'next', resultDirName: 'RepoResults123', prngSeed: 'testSeed', + isGo: false, }); // Remove all references to the base path so that snapshot pass successfully. @@ -198,7 +199,7 @@ describe("main", () => { }; await mainAsync({ - testType: "github", + testType: "scheduled", tmpfs: false, entrypoint: 'tsserver', diagnosticOutput: false, @@ -210,6 +211,7 @@ describe("main", () => { newTsNpmVersion: 'next', resultDirName: 'RepoResults123', prngSeed: 'testSeed', + isGo: false }); // Remove all references to the base path so that snapshot pass successfully. diff --git a/test/tsconfig.json b/test/tsconfig.json index 9992760..68b48df 100644 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -3,6 +3,7 @@ "compilerOptions": { "outDir": "dist", // Output files are ignored, but them in a folder for tidiness "emitDeclarationOnly": true, + "types": ["jest"] }, "references": [ { "path": "../src" },