JUnit Result Writer

.gitignore

@@ -1,2 +1,3 @@
+junit*.xml
 /framework-tests
 /example-tests

Makefile

@@ -28,7 +28,11 @@ unit:
 	go test ./...
 
 integration: build
-	./framework-tests run-suite framework
+ifeq ($(OPENSHIFT_CI), true)
+	./framework-tests run-suite openshift-tests-extension/framework --junit-path $(ARTIFACT_DIR)/junit_$(shell date +%Y%m%d-%H%M%S).xml
+else
+	./framework-tests run-suite openshift-tests-extension/framework
+endif
 
 lint:
 	./hack/go-lint.sh run ./...

cmd/framework-tests/main.go

@@ -20,7 +20,7 @@ func main() {
 	registry := e.NewRegistry()
 	ext := e.NewExtension("openshift", "framework", "default")
 	ext.AddSuite(e.Suite{
-		Name: "framework",
+		Name: "openshift-tests-extension/framework",
 	})
 
 	// If using Ginkgo, build test specs automatically

pkg/cmd/cmdrun/runsuite.go

@@ -17,10 +17,12 @@ func NewRunSuiteCommand(registry *extension.Registry) *cobra.Command {
 		componentFlags   *flags.ComponentFlags
 		outputFlags      *flags.OutputFlags
 		concurrencyFlags *flags.ConcurrencyFlags
+		junitPath        string
 	}{
 		componentFlags:   flags.NewComponentFlags(),
 		outputFlags:      flags.NewOutputFlags(),
 		concurrencyFlags: flags.NewConcurrencyFlags(),
+		junitPath:        "",
 	}
 
 	cmd := &cobra.Command{
@@ -35,29 +37,47 @@ func NewRunSuiteCommand(registry *extension.Registry) *cobra.Command {
 			if len(args) != 1 {
 				return fmt.Errorf("must specify one suite name")
 			}
-
-			w, err := extensiontests.NewResultWriter(os.Stdout, extensiontests.ResultFormat(opts.outputFlags.Output))
+			suite, err := ext.GetSuite(args[0])
 			if err != nil {
-				return err
+				return errors.Wrapf(err, "couldn't find suite: %s", args[0])
 			}
-			defer w.Flush()
 
-			suite, err := ext.GetSuite(args[0])
+			compositeWriter := extensiontests.NewCompositeResultWriter()
+			defer func() {
+				if err = compositeWriter.Flush(); err != nil {
+					fmt.Fprintf(os.Stderr, "failed to write results: %v\n", err)
+				}
+			}()
+
+			// JUnit writer if needed
+			if opts.junitPath != "" {
+				junitWriter, err := extensiontests.NewJUnitResultWriter(opts.junitPath, suite.Name)
+				if err != nil {
+					return errors.Wrap(err, "couldn't create junit writer")
+				}
+				compositeWriter.AddWriter(junitWriter)
+			}
+
+			// JSON writer
+			jsonWriter, err := extensiontests.NewJSONResultWriter(os.Stdout,
+				extensiontests.ResultFormat(opts.outputFlags.Output))
 			if err != nil {
-				return errors.Wrapf(err, "couldn't find suite: %s", args[0])
+				return err
 			}
+			compositeWriter.AddWriter(jsonWriter)
 
 			specs, err := ext.GetSpecs().Filter(suite.Qualifiers)
 			if err != nil {
 				return errors.Wrap(err, "couldn't filter specs")
 			}
 
-			return specs.Run(w, opts.concurrencyFlags.MaxConcurency)
+			return specs.Run(compositeWriter, opts.concurrencyFlags.MaxConcurency)
 		},
 	}
 	opts.componentFlags.BindFlags(cmd.Flags())
 	opts.outputFlags.BindFlags(cmd.Flags())
 	opts.concurrencyFlags.BindFlags(cmd.Flags())
+	cmd.Flags().StringVarP(&opts.junitPath, "junit-path", "j", opts.junitPath, "write results to junit XML")
 
 	return cmd
 }

pkg/cmd/cmdrun/runtest.go

@@ -63,7 +63,7 @@ func NewRunTestCommand(registry *extension.Registry) *cobra.Command {
 				return err
 			}
 
-			w, err := extensiontests.NewResultWriter(os.Stdout, extensiontests.ResultFormat(opts.outputFlags.Output))
+			w, err := extensiontests.NewJSONResultWriter(os.Stdout, extensiontests.ResultFormat(opts.outputFlags.Output))
 			if err != nil {
 				return err
 			}

pkg/extension/extensiontests/result.go

@@ -1,7 +1,60 @@
 package extensiontests
 
+import (
+	"fmt"
+	"strings"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/junit"
+)
+
 func (results ExtensionTestResults) Walk(walkFn func(*ExtensionTestResult)) {
 	for i := range results {
 		walkFn(results[i])
 	}
 }
+
+func (result ExtensionTestResult) ToJUnit() *junit.TestCase {
+	tc := &junit.TestCase{
+		Name:     result.Name,
+		Duration: float64(result.Duration) / 1000.0,
+	}
+	switch result.Result {
+	case ResultFailed:
+		tc.FailureOutput = &junit.FailureOutput{
+			Message: result.Error,
+			Output:  result.Error,
+		}
+	case ResultSkipped:
+		tc.SkipMessage = &junit.SkipMessage{
+			Message: strings.Join(result.Details, "\n"),
+		}
+	case ResultPassed:
+		tc.SystemOut = result.Output
+	}
+
+	return tc
+}
+
+func (results ExtensionTestResults) ToJUnit(suiteName string) junit.TestSuite {
+	suite := junit.TestSuite{
+		Name: suiteName,
+	}
+
+	results.Walk(func(result *ExtensionTestResult) {
+		suite.NumTests++
+		switch result.Result {
+		case ResultFailed:
+			suite.NumFailed++
+		case ResultSkipped:
+			suite.NumSkipped++
+		case ResultPassed:
+			// do nothing
+		default:
+			panic(fmt.Sprintf("unknown result type: %s", result.Result))
+		}
+
+		suite.TestCases = append(suite.TestCases, result.ToJUnit())
+	})
+
+	return suite
+}

pkg/extension/extensiontests/result_writer.go

@@ -2,38 +2,136 @@ package extensiontests
 
 import (
 	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"io"
+	"os"
+	"sync"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/junit"
 )
 
+// ResultWriter is an interface for recording ExtensionTestResults in a particular format.
+// Implementations must be threadsafe.
+type ResultWriter interface {
+	Write(*ExtensionTestResult)
+	Flush() error
+}
+
+type CompositeResultWriter struct {
+	writers []ResultWriter
+}
+
+func NewCompositeResultWriter(writers ...ResultWriter) *CompositeResultWriter {
+	return &CompositeResultWriter{
+		writers: writers,
+	}
+}
+
+func (w *CompositeResultWriter) AddWriter(writer ResultWriter) {
+	w.writers = append(w.writers, writer)
+}
+
+func (w *CompositeResultWriter) Write(res *ExtensionTestResult) {
+	for _, writer := range w.writers {
+		writer.Write(res)
+	}
+}
+
+func (w *CompositeResultWriter) Flush() error {
+	var errs []error
+	for _, writer := range w.writers {
+		if err := writer.Flush(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if len(errs) > 0 {
+		return fmt.Errorf("encountered errors from writers: %v", errs)
+	}
+
+	return nil
+}
+
+type JUnitResultWriter struct {
+	lock      sync.Mutex
+	testSuite *junit.TestSuite
+	out       *os.File
+	suiteName string
+	path      string
+	results   ExtensionTestResults
+}
+
+func NewJUnitResultWriter(path, suiteName string) (ResultWriter, error) {
+	file, err := os.Create(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return &JUnitResultWriter{
+		testSuite: &junit.TestSuite{
+			Name: suiteName,
+		},
+		out:       file,
+		suiteName: suiteName,
+		path:      path,
+	}, nil
+}
+
+func (w *JUnitResultWriter) Write(res *ExtensionTestResult) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+	w.results = append(w.results, res)
+}
+
+func (w *JUnitResultWriter) Flush() error {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+	data, err := xml.Marshal(w.results.ToJUnit(w.suiteName))
+	if err != nil {
+		panic(err)
+	}
+	if _, err := w.out.Write(data); err != nil {
+		return err
+	}
+	if err := w.out.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 type ResultFormat string
 
 var (
 	JSON  ResultFormat = "json"
 	JSONL ResultFormat = "jsonl"
 )
 
-type ResultWriter struct {
+type JSONResultWriter struct {
+	lock    sync.Mutex
 	out     io.Writer
 	format  ResultFormat
 	results ExtensionTestResults
 }
 
-func NewResultWriter(out io.Writer, format ResultFormat) (*ResultWriter, error) {
+func NewJSONResultWriter(out io.Writer, format ResultFormat) (*JSONResultWriter, error) {
 	switch format {
 	case JSON, JSONL:
 	// do nothing
 	default:
 		return nil, fmt.Errorf("unsupported result format: %s", format)
 	}
 
-	return &ResultWriter{
+	return &JSONResultWriter{
 		out:    out,
 		format: format,
 	}, nil
 }
 
-func (w *ResultWriter) Write(result *ExtensionTestResult) {
+func (w *JSONResultWriter) Write(result *ExtensionTestResult) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
 	switch w.format {
 	case JSONL:
 		// JSONL gets written to out as we get the items
@@ -47,15 +145,20 @@ func (w *ResultWriter) Write(result *ExtensionTestResult) {
 	}
 }
 
-func (w *ResultWriter) Flush() {
+func (w *JSONResultWriter) Flush() error {
+	w.lock.Lock()
+	defer w.lock.Unlock()
 	switch w.format {
 	case JSONL:
 	// we already wrote it out
 	case JSON:
 		data, err := json.MarshalIndent(w.results, "", "  ")
 		if err != nil {
-			panic(err)
+			return err
 		}
-		fmt.Fprintf(w.out, "%s\n", string(data))
+		_, err = w.out.Write(data)
+		return err
 	}
+
+	return nil
 }

pkg/extension/extensiontests/spec.go

@@ -39,7 +39,7 @@ func (specs ExtensionTestSpecs) Names() []string {
 	return names
 }
 
-func (specs ExtensionTestSpecs) Run(w *ResultWriter, maxConcurrent int) error {
+func (specs ExtensionTestSpecs) Run(w ResultWriter, maxConcurrent int) error {
 	queue := make(chan *ExtensionTestSpec)
 	failures := atomic.Int64{}