fix: clarify Python pytest failure messages

executor.js

@@ -114,6 +114,13 @@ function configureExecution(language, code, uniqueDir) {
   return config;
 }
 
+function buildPytestInvocation(uniqueDir) {
+  return {
+    command: 'pytest',
+    args: ['-vv', path.join(uniqueDir, 'test_program.py')],
+  };
+}
+
 /**
  * Writes the provided code to a file.
  * @param {string} uniqueDir - The directory to write the file.
@@ -303,6 +310,35 @@ function extractPytestErrorMessage(output, stderr = '') {
   return 'Pytest error during collection or execution';
 }
 
+function hasCustomAssertionMessage(assertionSource) {
+  const expression = assertionSource.replace(/^>\s*assert\s+/, '');
+  let depth = 0;
+  let quote;
+  let escaped = false;
+
+  for (const character of expression) {
+    if (quote) {
+      if (escaped) {
+        escaped = false;
+      } else if (character === '\\') {
+        escaped = true;
+      } else if (character === quote) {
+        quote = undefined;
+      }
+    } else if (character === '"' || character === "'") {
+      quote = character;
+    } else if (character === '(' || character === '[' || character === '{') {
+      depth += 1;
+    } else if (character === ')' || character === ']' || character === '}') {
+      depth -= 1;
+    } else if (character === ',' && depth === 0) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 function parsePytestOutput(stdout = '', stderr = '', exitCode = null) {
   const summary = extractPytestSummary(stdout);
   const rawout = buildRawOutput(stdout, stderr);
@@ -311,48 +347,100 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) {
   const errors = extractPytestCount(summary, 'error(?:s)?');
   const no_tests_collected = exitCode === 5 || /\bno tests ran\b/.test(summary);
   const failures = [];
+  let currentFailure;
 
-  const failureBlocks = stdout.split(/={10,} FAILURES ={10,}/)[1]?.split(/={10,}/)[0] || '';
-  const matches = [...failureBlocks.matchAll(
-    /_{5,}\s*(.*?)\s*_{5,}[\s\S]*?>\s*assert\s+(.*?)\s*?\nE\s+assert\s+(.*?)\s*?(?:\nE\s+\+\s+where\s+(.*?)\s+=)?/g
-  )];
+  const addFailure = () => {
+    if (!currentFailure?.errorLines.length) {
+      return;
+    }
 
-  matches.forEach((match, index) => {
-    const test_case = match[1]?.trim() || `Test ${index + 1}`;
-    const assertionLine = match[2]?.trim();
-    const failedExpr = match[3]?.trim();
-    const evaluated = match[4]?.trim() || '';
+    const messageLines = currentFailure.errorLines.map((line) =>
+      line.replace(/^\s*E {0,7}/, '')
+    );
+    const firstMessageLine = messageLines[0];
+    const usesCustomAssertionMessage =
+      hasCustomAssertionMessage(currentFailure.assertionSource) &&
+      firstMessageLine.startsWith('AssertionError:');
+    const comparison = firstMessageLine
+      .replace(/^AssertionError:\s*/, '')
+      .match(/^assert\s+(.+?)\s*==\s*(.+)$/);
 
     failures.push({
-      test_case,
-      expected: failedExpr.split('==')[1]?.trim() || '',
-      received: evaluated || failedExpr.split('==')[0]?.trim(),
-      error_message: `Assertion failed: ${assertionLine}`,
+      test_case: currentFailure.testCase || `Test ${failures.length + 1}`,
+      expected: comparison?.[2]?.trim() || '',
+      received: comparison?.[1]?.trim() || '',
+      error_message: usesCustomAssertionMessage
+        ? [firstMessageLine.replace(/^AssertionError:\s*/, ''), ...messageLines.slice(1)].join('\n')
+        : messageLines.join('\n'),
       rawout,
+      isError: currentFailure.isError,
     });
-  });
+  };
+
+  for (const line of stdout.split(/\r?\n/)) {
+    const failureHeader = line.match(/^_{5,}\s*(.*?)\s*_{5,}\s*$/);
+    if (failureHeader) {
+      addFailure();
+      currentFailure = {
+        testCase: failureHeader[1].trim(),
+        assertionSource: '',
+        errorLines: [],
+        finishedErrors: false,
+        isError: /^ERROR\b/.test(failureHeader[1].trim()),
+      };
+      continue;
+    }
+
+    if (!currentFailure) {
+      continue;
+    }
 
-  if (failed_tests > 0 && failures.length === 0) {
+    if (/^={3,}|^!{3,}/.test(line)) {
+      addFailure();
+      currentFailure = undefined;
+      continue;
+    }
+
+    if (/^>\s*assert\b/.test(line)) {
+      currentFailure.assertionSource = line;
+    }
+
+    if (/^\s*E(?:\s|$)/.test(line) && !currentFailure.finishedErrors) {
+      currentFailure.errorLines.push(line);
+    } else if (currentFailure.errorLines.length) {
+      currentFailure.finishedErrors = true;
+    }
+  }
+  addFailure();
+
+  const parsedFailures = failures.filter((failure) => !failure.isError).length;
+  const parsedErrors = failures.length - parsedFailures;
+
+  if (failed_tests > parsedFailures) {
     failures.push({
       test_case: extractPytestShortSummaryTarget(stdout, 'FAILED') || 'pytest assertion failure',
       expected: '',
       received: '',
       error_message: 'Pytest reported one or more failed assertions',
       rawout,
+      isError: false,
     });
   }
 
   let runtime_error = '';
 
   if (errors > 0) {
     runtime_error = extractPytestErrorMessage(stdout, stderr);
-    failures.push({
-      test_case: extractPytestShortSummaryTarget(stdout, 'ERROR') || 'pytest collection/execution',
-      expected: '',
-      received: '',
-      error_message: runtime_error,
-      rawout,
-    });
+    if (errors > parsedErrors) {
+      failures.push({
+        test_case: extractPytestShortSummaryTarget(stdout, 'ERROR') || 'pytest collection/execution',
+        expected: '',
+        received: '',
+        error_message: runtime_error,
+        rawout,
+        isError: true,
+      });
+    }
   } else if (no_tests_collected) {
     runtime_error = 'Pytest did not collect any tests';
     failures.push({
@@ -361,6 +449,7 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) {
       received: '0 collected tests',
       error_message: runtime_error,
       rawout,
+      isError: true,
     });
   }
 
@@ -371,11 +460,45 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) {
     errors,
     no_tests_collected,
     exit_code: exitCode,
-    failure_details: failures,
+    failure_details: failures.map(({ isError, ...failure }) => failure),
     runtime_error,
   };
 }
 
+function buildPythonTestResponse(response, output) {
+  const testResults = parsePytestOutput(
+    output.stdout,
+    output.stderr,
+    output.exitCode ?? null
+  );
+  const keepGenericRuntimeError = output.exitCode == null || (
+    output.exitCode !== 0 &&
+    output.exitCode !== 1 &&
+    output.exitCode !== 5 &&
+    testResults.errors === 0
+  );
+  const runtime_error = testResults.runtime_error || (
+    keepGenericRuntimeError ? response.runtime_error : ''
+  );
+  const hasUnexpectedPytestExecutionError = Boolean(runtime_error) && (
+    testResults.failed === 0 &&
+    testResults.errors === 0 &&
+    !testResults.no_tests_collected
+  );
+
+  return {
+    ...response,
+    ...testResults,
+    runtime_error,
+    state: (
+      testResults.failed === 0 &&
+      testResults.errors === 0 &&
+      !testResults.no_tests_collected &&
+      !hasUnexpectedPytestExecutionError
+    ) ? 'passed' : 'failed',
+  };
+}
+
 function parseCppTestOutput(output, stdout = '', stderr = '') {
   output = output.toString();
   let total_tests = 0;
@@ -488,8 +611,9 @@ async function executeCode(language, code, stdin, expectedOutput, runTests = fal
     let output;
     if (runTests && testCode) {
       if (language.toLowerCase() === 'python') {
-        executionConfig.runCommand = 'pytest';
-        executionConfig.runArgs = [path.join(uniqueDir, 'test_program.py')];
+        const pytestInvocation = buildPytestInvocation(uniqueDir);
+        executionConfig.runCommand = pytestInvocation.command;
+        executionConfig.runArgs = pytestInvocation.args;
       } else if (language.toLowerCase() === 'cpp') {
         executionConfig.runCommand = path.join(uniqueDir, 'runner');
         executionConfig.runArgs = [];
@@ -549,29 +673,7 @@ async function executeCode(language, code, stdin, expectedOutput, runTests = fal
 
     if (runTests && testCode) {
       if (language.toLowerCase() === 'python') {
-        const testResults = parsePytestOutput(output.stdout, output.stderr, output.exitCode ?? null);
-        const keepGenericRuntimeError = output.exitCode == null || (
-          output.exitCode !== 0 &&
-          output.exitCode !== 1 &&
-          output.exitCode !== 5 &&
-          testResults.errors === 0
-        );
-        const runtime_error = testResults.runtime_error || (keepGenericRuntimeError ? response.runtime_error : '');
-        const hasUnexpectedPytestExecutionError = Boolean(runtime_error) && (
-          testResults.failed === 0 &&
-          testResults.errors === 0 &&
-          !testResults.no_tests_collected
-        );
-
-        response = { ...response, ...testResults };
-        response.runtime_error = runtime_error;
-        response.state = (
-          testResults.failed === 0 &&
-          testResults.errors === 0 &&
-          !testResults.no_tests_collected &&
-          !hasUnexpectedPytestExecutionError
-        ) ? 'passed' : 'failed';
-        return response;
+        return buildPythonTestResponse(response, output);
       }
 
       if (language.toLowerCase() === 'cpp') {
@@ -762,4 +864,9 @@ async function cleanupDir(dirPath) {
   }
 }
 
-module.exports = { executeCode };
+module.exports = {
+  buildPytestInvocation,
+  buildPythonTestResponse,
+  executeCode,
+  parsePytestOutput,
+};

package.json

@@ -3,7 +3,9 @@
   "version": "1.0.0",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "npm run test:unit && npm run test:integration",
+    "test:unit": "node test/executor.test.js",
+    "test:integration": "node test/pytest-integration.test.js"
   },
   "keywords": [],
   "author": "",