Update interface and use @exercism/static-analysis (#31)

- Add `test_code` when available
- Fix `output` when there is none (previously `""`, now `null`)

Author: SleeplessByte

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 2.2.0
+
+- Add `test_code` when available
+- Fix `output` when there is none (previously `""`, now `null`)
+
 ## 2.1.0
 
 - Add `output` per test (user `console.log`)

README.md

@@ -29,7 +29,7 @@ You'll want these `:dev` variants because it will _build_ the required code (it
 
 You can also manually build using `yarn` or `yarn build`, and then run the script directly: `./bin/run.sh arg1 arg2 arg3`.
 
-## Running the Tests
+## Running the Solution's Tests
 
 To run a solution's tests, do the following:
 
@@ -80,7 +80,7 @@ Find the output at:
 
 As you can see, it will be copied to a local directory. It's up to you to clean-up this directory.
 
-## Running the Tests in Docker container
+## Running the Solution's Tests in Docker container
 
 _This script is provided for testing purposes_
 
@@ -93,9 +93,9 @@ To run a solution's test in the Docker container, do the following:
 
 The `package.json` needs to be in-sync with the [`javascript` track `package.json`][git-javascript].
 
-### Known issues
+### Testing
 
-- The output format of the tests still does not conform to the [exercism automated tests][git-automated-tests] standard.
+Running the tests of the test-runner itself can be achieved by using the `test` script from `package.json`. The tests delegate to the _build output_, which is why `yarn test` first calls `yarn build` before running `jest`. **The tests take over a minute to run on a decent machine**.
 
 [web-exercism]: https://exercism.io
 [git-automated-tests]: https://github.com/exercism/automated-tests

bin/run.sh

@@ -68,9 +68,11 @@ else
 fi
 
 # Forces a trailing slash
+INPUT="${INPUT//\\//}"
 INPUT="${INPUT%/}/"
 
 # Forces a trailing slash
+OUTPUT="${OUTPUT//\\//}"
 OUTPUT="${OUTPUT%/}/"
 
 set -euo pipefail

package.json

@@ -2,7 +2,7 @@
   "name": "@exercism/javascript-test-runner",
   "description": "Automated Test runner for exercism solutions in Javascript.",
   "author": "Derk-Jan Karrenbeld <derk-jan+github@karrenbeld.info>",
-  "version": "2.0.0",
+  "version": "2.2.0",
   "license": "AGPL-3.0-or-later",
   "repository": {
     "type": "git",
@@ -22,7 +22,8 @@
     "prebuild": "rimraf ./dist",
     "build": "yarn tsc --project ./src/tsconfig.json --outDir ./dist",
     "watch": "yarn build -w",
-    "prepublish": "yarn test",
+    "prepare": "yarn build",
+    "prepublishOnly": "yarn test:bare && yarn lint",
     "lint": "yarn eslint . --ext ts,js,tsx,jsx,mjs -c .eslintrc",
     "test": "yarn build && yarn test:bare",
     "test:bare": "jest --roots test --testPathIgnorePatterns=\"fixtures/\""
@@ -33,6 +34,8 @@
     "@babel/node": "^7.12.10",
     "@babel/preset-env": "^7.12.11",
     "@babel/preset-typescript": "^7.12.7",
+    "@exercism/static-analysis": "^0.4.1",
+    "@typescript-eslint/typescript-estree": "^4.11.1",
     "babel-jest": "^26.6.3",
     "chalk": "^4.1.0",
     "jest": "^26.6.3",
@@ -43,12 +46,11 @@
   },
   "devDependencies": {
     "@types/jest": "^26.0.19",
-    "@types/node": "^14.14.16",
-    "@typescript-eslint/eslint-plugin": "^4.11.0",
-    "@typescript-eslint/parser": "^4.11.0",
-    "@typescript-eslint/typescript-estree": "^4.11.0",
+    "@types/node": "^14.14.19",
+    "@typescript-eslint/eslint-plugin": "^4.11.1",
+    "@typescript-eslint/parser": "^4.11.1",
     "babel-eslint": "^10.1.0",
-    "eslint": "^7.16.0",
+    "eslint": "^7.17.0",
     "eslint-config-prettier": "^7.1.0",
     "eslint-plugin-import": "^2.22.1",
     "eslint-plugin-jest": "^24.1.3",

src/output.ts

@@ -1,13 +1,21 @@
-import path from 'path'
-import fs from 'fs'
-
+import {
+  AstParser,
+  ParsedSource,
+} from '@exercism/static-analysis/dist/AstParser'
+import {
+  extractTests,
+  TestCase,
+} from '@exercism/static-analysis/dist/extracts/extract_tests'
+import { FileInput } from '@exercism/static-analysis/dist/input/FileInput'
 import { ConsoleBuffer } from '@jest/console'
 import {
   AggregatedResult,
   AssertionResult,
   TestResult,
 } from '@jest/test-result'
 import { Config } from '@jest/types'
+import fs from 'fs'
+import path from 'path'
 
 interface OutputInterface {
   status: 'fail' | 'pass' | 'error'
@@ -20,6 +28,7 @@ interface OutputTestInterface {
   status: 'fail' | 'pass' | 'error'
   message: string
   output: string | null
+  test_code: string
 }
 
 export class Output {
@@ -64,11 +73,62 @@ export class Output {
       }
     }
 
+    const parsedSources: Record<
+      string,
+      {
+        program: ParsedSource['program']
+        source: ParsedSource['source']
+        tests: Record<string, TestCase>
+      }
+    > = {}
+
+    // Every tested test file is indexed and parsed
+    this.results.tests
+      .map((test) => test.test_code)
+      .filter((path, index, self) => self.indexOf(path) === index)
+      .forEach((file) => {
+        try {
+          const [{ program, source }] = AstParser.ANALYZER.parseSync(
+            fs.readFileSync(file).toString()
+          )
+          const tests = extractTests(program)
+
+          parsedSources[file] = {
+            program,
+            source,
+            tests: tests.reduce((results, item) => {
+              results[item.name(' > ')] = item
+              results[item.name(' ' as ' > ')] = item
+              return results
+            }, {} as Record<string, TestCase>),
+          }
+        } catch (err) {
+          console.error(
+            `When trying to parse ${file}, the following error occurred`,
+            err
+          )
+        }
+      })
+
+    // Extract the test code, if possible
+    const tests = this.results.tests.map((test) => {
+      const parsedSource = parsedSources[test.test_code]
+      if (!parsedSource) {
+        return { ...test, test_code: null }
+      }
+
+      const testCase = parsedSource.tests[test.name]
+      if (!testCase) {
+        return { ...test, test_code: null }
+      }
+
+      return { ...test, test_code: testCase.testCode(parsedSource.source) }
+    })
+
     // Re-order the output so that tests output shows below main output
-    const { status, message, tests } = this.results
+    const { status, message } = this.results
 
     const artifact = JSON.stringify({ status, message, tests }, undefined, 2)
-
     fs.writeFileSync(this.outputFile, artifact)
   }
 
@@ -143,8 +203,10 @@ export class Output {
         return {
           ...withoutOutput,
           output: isFirstFailure
-            ? [consoleOutputs[''], outputMessage].filter(Boolean).join('\n')
+            ? [consoleOutputs[''], outputMessage].filter(Boolean).join('\n') ||
+              null
             : outputMessage,
+          test_code: specFilePath,
         }
       })
     )

test/.eslintignore

@@ -0,0 +1 @@
+fixtures/*

test/fixtures/two-fer/error/malformed_tests/two-fer.js

@@ -0,0 +1 @@
+export const twoFer = (name = 'you') => `One for ${name}, one for me.`

test/fixtures/two-fer/error/malformed_tests/two-fer.spec.js

@@ -0,0 +1 @@
+describe('twoFer()', t('another name given', () => {