feat(dialtesting): add SSL task and sanitize browser runner errors

dialtesting/browser.go

@@ -42,6 +42,8 @@ const (
 	optionLightpandaPathCamel = "lightpandaPath"
 	optionChromePath          = "chrome_path"
 	optionChromePathCamel     = "chromePath"
+
+	browserSystemErrorMessage = "Browser dial testing system error, please check agent logs"
 )
 
 type BrowserTask struct {
@@ -597,8 +599,9 @@ func (t *BrowserTask) getResults() (tags map[string]string, fields map[string]in
 		fields["message"] = "success"
 	} else {
 		reasons, _ := t.checkResult()
-		fields["fail_reason"] = strings.Join(reasons, ";")
-		fields["message"] = strings.Join(reasons, ";")
+		displayReasons := t.displayReasons(reasons)
+		fields["fail_reason"] = strings.Join(displayReasons, ";")
+		fields["message"] = strings.Join(displayReasons, ";")
 		fields["failure_type"] = t.result.FailureType
 		if fields["failure_type"] == "" && t.reqError != "" {
 			fields["failure_type"] = "config_error"
@@ -607,15 +610,16 @@ func (t *BrowserTask) getResults() (tags map[string]string, fields map[string]in
 	if len(t.result.TraceIDs) > 0 {
 		fields["trace_id"] = t.result.TraceIDs[0]
 	}
-	if steps, err := json.Marshal(compactBrowserSteps(t.result.Steps)); err == nil {
+	displayError := t.displayRunnerError()
+	if steps, err := json.Marshal(sanitizeBrowserSteps(compactBrowserSteps(t.result.Steps), displayError)); err == nil {
 		fields["steps"] = string(steps)
 	}
 	if len(result.retryRecords) > 0 {
-		if data, err := json.Marshal(result.retryRecords); err == nil {
+		if data, err := json.Marshal(sanitizeBrowserRetryRecords(result.retryRecords, displayError)); err == nil {
 			fields["retry_records"] = string(data)
 		}
 	} else if len(t.result.RetryRecords) > 0 {
-		if data, err := json.Marshal(t.result.RetryRecords); err == nil {
+		if data, err := json.Marshal(sanitizeBrowserRetryRecords(t.result.RetryRecords, displayError)); err == nil {
 			fields["retry_records"] = string(data)
 		}
 	}
@@ -683,6 +687,89 @@ func compactBrowserSteps(steps []browserDialStep) []browserDialStep {
 	return compact
 }
 
+func (t *BrowserTask) displayReasons(reasons []string) []string {
+	if displayError := t.displayRunnerError(); displayError != "" {
+		raw := strings.Join(reasons, ";")
+		if raw == "" {
+			raw = t.runnerErrorText()
+		}
+		logger.Warnf("browser runner error sanitized for dialtesting result: %s", raw)
+		return []string{displayError}
+	}
+	return reasons
+}
+
+func (t *BrowserTask) displayRunnerError() string {
+	if !t.isRunnerError() {
+		return ""
+	}
+	return browserSystemErrorMessage
+}
+
+func (t *BrowserTask) isRunnerError() bool {
+	return strings.EqualFold(t.result.FailReason, "runner_error")
+}
+
+func (t *BrowserTask) runnerErrorText() string {
+	parts := []string{t.reqError, t.result.FailReason}
+	if t.result.Error != nil {
+		parts = append(parts, t.result.Error.Message)
+	}
+	for _, step := range t.result.Steps {
+		if step.Error != nil {
+			parts = append(parts, step.Error.Message)
+		}
+	}
+	for _, record := range t.result.RetryRecords {
+		parts = append(parts, record.Message)
+	}
+	return strings.Join(parts, "\n")
+}
+
+func sanitizeBrowserSteps(steps []browserDialStep, displayError string) []browserDialStep {
+	if displayError == "" || len(steps) == 0 {
+		return steps
+	}
+
+	out := make([]browserDialStep, len(steps))
+	copy(out, steps)
+	for i := range out {
+		if out[i].Error == nil {
+			continue
+		}
+		errInfo := *out[i].Error
+		errInfo.Message = displayError
+		errInfo.Stack = ""
+		out[i].Error = &errInfo
+	}
+	return out
+}
+
+func sanitizeBrowserRetryRecords(records []browserRetryRecord, displayError string) []browserRetryRecord {
+	if len(records) == 0 {
+		return records
+	}
+
+	out := make([]browserRetryRecord, len(records))
+	copy(out, records)
+	for i := range out {
+		if !isBrowserRunnerRetryRecord(out[i]) {
+			continue
+		}
+		if displayError != "" {
+			out[i].Message = displayError
+		} else {
+			out[i].Message = browserSystemErrorMessage
+		}
+	}
+	return out
+}
+
+func isBrowserRunnerRetryRecord(record browserRetryRecord) bool {
+	return strings.EqualFold(record.FailReason, "runner_error") ||
+		strings.EqualFold(record.FailureType, "runner_error")
+}
+
 func (t *BrowserTask) check() error {
 	if strings.TrimSpace(t.BrowserConfig) == "" {
 		return errors.New("browser_config should not be empty")

dialtesting/browser_error_test.go

@@ -0,0 +1,165 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the MIT License.
+// This product includes software developed at Guance Cloud (https://www.guance.com/).
+// Copyright 2021-present Guance, Inc.
+
+package dialtesting
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBrowserTaskDisplayRunnerError(t *testing.T) {
+	raw := "lightpanda exited before CDP was ready: exit status 1\n" +
+		"/root/.local/bin/lightpanda: /lib64/libm.so.6: version GLIBC_2.27 not found\n" +
+		"/root/.local/bin/lightpanda: /lib64/libc.so.6: version GLIBC_2.34 not found"
+	task := &BrowserTask{
+		result: browserDialRun{
+			FailReason:  "runner_error",
+			FailureType: "browser_error",
+			Error:       &browserDialError{Name: "Error", Message: raw, Stack: "/root/.local/bin/lightpanda"},
+			Steps: []browserDialStep{
+				{
+					Seq:    1,
+					Status: "FAIL",
+					Error:  &browserDialError{Name: "Error", Message: raw, Stack: "/root/.local/bin/lightpanda"},
+				},
+			},
+			RetryRecords: []browserRetryRecord{
+				{Attempt: 1, Status: "FAIL", FailureType: "runner_error", Message: raw},
+			},
+		},
+	}
+
+	display := task.displayRunnerError()
+	assert.Equal(t, browserSystemErrorMessage, display)
+	assert.Equal(t, []string{display}, task.displayReasons([]string{"runner_error", raw}))
+
+	steps := sanitizeBrowserSteps(task.result.Steps, display)
+	assert.Len(t, steps, 1)
+	assert.Equal(t, display, steps[0].Error.Message)
+	assert.Empty(t, steps[0].Error.Stack)
+
+	records := sanitizeBrowserRetryRecords(task.result.RetryRecords, display)
+	assert.Len(t, records, 1)
+	assert.Equal(t, display, records[0].Message)
+
+	assertNoLeak(t, steps[0].Error.Message, "/root/.local/bin/lightpanda", "GLIBC_")
+	assertNoLeak(t, records[0].Message, "/root/.local/bin/lightpanda", "GLIBC_")
+}
+
+func TestSanitizeBrowserRetryRecordsKeepsNonRunnerMessages(t *testing.T) {
+	records := []browserRetryRecord{
+		{
+			Attempt:     1,
+			Status:      "FAIL",
+			FailReason:  "step_error",
+			FailureType: "assertion_failed",
+			Message:     "title assertion failed: expected Home",
+		},
+		{
+			Attempt:     2,
+			Status:      "FAIL",
+			FailReason:  "runner_error",
+			FailureType: "browser_error",
+			Message:     "lightpanda exited before CDP was ready: exit status 1",
+		},
+		{
+			Attempt:     3,
+			Status:      "FAIL",
+			FailureType: "runner_error",
+			Message:     "runner engine factory is not configured",
+		},
+	}
+
+	got := sanitizeBrowserRetryRecords(records, browserSystemErrorMessage)
+
+	assert.Len(t, got, 3)
+	assert.Equal(t, "title assertion failed: expected Home", got[0].Message)
+	assert.Equal(t, browserSystemErrorMessage, got[1].Message)
+	assert.Equal(t, browserSystemErrorMessage, got[2].Message)
+}
+
+func TestSanitizeBrowserRetryRecordsHidesRunnerErrorWithoutFinalRunnerError(t *testing.T) {
+	records := []browserRetryRecord{
+		{
+			Attempt:     1,
+			Status:      "FAIL",
+			FailureType: "runner_error",
+			Message:     "/opt/lightpanda: /lib64/libc.so.6: version GLIBC_2.34 not found",
+		},
+		{
+			Attempt: 2,
+			Status:  "OK",
+			Success: true,
+			Message: "success",
+		},
+	}
+
+	got := sanitizeBrowserRetryRecords(records, "")
+
+	assert.Len(t, got, 2)
+	assert.Equal(t, browserSystemErrorMessage, got[0].Message)
+	assert.Equal(t, "success", got[1].Message)
+	assertNoLeak(t, got[0].Message, "/opt/lightpanda", "GLIBC_")
+}
+
+func TestBrowserTaskDisplayRunnerErrorUsesUnifiedMessage(t *testing.T) {
+	rawMessages := []string{
+		"no lightpanda executable found; set LIGHTPANDA_EXECUTABLE_PATH or install lightpanda in PATH",
+		"fork/exec /opt/lightpanda: permission denied",
+		"/lib64/libc.so.6: version GLIBC_2.34 not found",
+		"lightpanda CDP server did not become ready at http://127.0.0.1:9222",
+	}
+
+	for _, raw := range rawMessages {
+		t.Run(raw, func(t *testing.T) {
+			task := &BrowserTask{
+				result: browserDialRun{
+					FailReason: "runner_error",
+					Error:      &browserDialError{Message: raw},
+				},
+			}
+
+			assert.Equal(t, browserSystemErrorMessage, task.displayRunnerError())
+		})
+	}
+}
+
+func TestBrowserTaskDisplayRunnerErrorKeepsNonRunnerError(t *testing.T) {
+	task := &BrowserTask{
+		result: browserDialRun{
+			FailReason:  "selector_not_found",
+			FailureType: "selector_not_found",
+			Error:       &browserDialError{Message: "selector not found: #login"},
+		},
+	}
+
+	assert.Empty(t, task.displayRunnerError())
+	assert.Equal(t, []string{"selector not found: #login"}, task.displayReasons([]string{"selector not found: #login"}))
+}
+
+func TestBrowserTaskDisplayRunnerErrorKeepsConfigErrorMentioningLightpanda(t *testing.T) {
+	task := &BrowserTask{
+		reqError: "advance_options engine should be chrome or lightpanda",
+		result: browserDialRun{
+			FailureType: "config_error",
+		},
+	}
+
+	assert.Empty(t, task.displayRunnerError())
+	assert.Equal(t,
+		[]string{"advance_options engine should be chrome or lightpanda"},
+		task.displayReasons([]string{"advance_options engine should be chrome or lightpanda"}),
+	)
+}
+
+func assertNoLeak(t *testing.T, value string, leaks ...string) {
+	t.Helper()
+	for _, leak := range leaks {
+		assert.False(t, strings.Contains(value, leak))
+	}
+}

dialtesting/browser_test.go

@@ -109,6 +109,25 @@ func TestBrowserTaskRunEmbeddedFailure(t *testing.T) {
 	assert.Contains(t, fields["steps"], "title")
 }
 
+func TestBrowserTaskRunEmbeddedScriptError(t *testing.T) {
+	browserTask := newBrowserTaskForTest()
+	browserTask.BrowserConfig = "name: browser\nsteps:\n  - action: eval\n    text: window.fail()\n"
+	task, err := NewTask("", browserTask)
+	require.NoError(t, err)
+
+	stubBrowserEngineWithFactory(t, func(context.Context, browserrunner.EngineOptions) (browserrunner.Engine, error) {
+		return &fakeBrowserEngine{evalErr: errors.New("eval boom")}, nil
+	})
+
+	require.NoError(t, task.Run())
+	_, fields := task.GetResults()
+	assert.Equal(t, int64(-1), fields["success"])
+	assert.Contains(t, fields["fail_reason"], "step_error")
+	assert.Equal(t, "script_error", fields["failure_type"])
+	assert.Contains(t, fields["message"], "eval boom")
+	assert.NotEqual(t, browserSystemErrorMessage, fields["message"])
+}
+
 func TestBrowserTaskRunEmbeddedEngineError(t *testing.T) {
 	browserTask := newBrowserTaskForTest()
 	task, err := NewTask("", browserTask)
@@ -125,7 +144,22 @@ func TestBrowserTaskRunEmbeddedEngineError(t *testing.T) {
 	require.NoError(t, task.Run())
 	_, fields := task.GetResults()
 	assert.Equal(t, int64(-1), fields["success"])
-	assert.Contains(t, fields["message"], "start chrome failed")
+	assert.Equal(t, browserSystemErrorMessage, fields["message"])
+	assert.NotContains(t, fields["message"], "start chrome failed")
+}
+
+func TestBrowserTaskRunInvalidEngineKeepsConfigError(t *testing.T) {
+	browserTask := newBrowserTaskForTest()
+	browserTask.AdvanceOptions = &BrowserAdvanceOption{Engine: "firefox"}
+	task, err := NewTask("", browserTask)
+	require.NoError(t, err)
+
+	require.NoError(t, task.Run())
+	_, fields := task.GetResults()
+	assert.Equal(t, int64(-1), fields["success"])
+	assert.Equal(t, "config_error", fields["failure_type"])
+	assert.Contains(t, fields["message"], "advance_options engine should be chrome or lightpanda")
+	assert.NotEqual(t, browserSystemErrorMessage, fields["message"])
 }
 
 func TestBrowserTaskRenderTemplate(t *testing.T) {
@@ -865,8 +899,9 @@ func TestBrowserTaskRunEmbedded(t *testing.T) {
 }
 
 type fakeBrowserEngine struct {
-	config *browserrunner.BrowserConfig
-	title  string
+	config  *browserrunner.BrowserConfig
+	title   string
+	evalErr error
 }
 
 func (e *fakeBrowserEngine) Close(context.Context) error { return nil }
@@ -892,7 +927,12 @@ func (e *fakeBrowserEngine) Text(context.Context, string) (string, error) {
 	return "Example Domain", nil
 }
 
-func (e *fakeBrowserEngine) Eval(context.Context, string) (string, error) { return "", nil }
+func (e *fakeBrowserEngine) Eval(context.Context, string) (string, error) {
+	if e.evalErr != nil {
+		return "", e.evalErr
+	}
+	return "", nil
+}
 
 func (e *fakeBrowserEngine) CaptureDOM(context.Context) (browserevidence.DomSnapshot, error) {
 	return browserevidence.DomSnapshot{}, nil