conformance: support OCI_REPORT_DIR (#473)

rogpeppe · web-flow · commit 3940529fe6c0 · 2023-10-16T08:16:59.000-05:00
This makes it possible to run the conformance tests in a read-only directory
without failing by specifying an output directory for the report files
instead of hard-coding it to the current directory (which may not be
writable, and _won't_ be writable if the tests are being run with
`go test github.com/opencontainers/distribution-spec/conformance`
rather than in a git checkout of the conformance module.

We recognize the value `none` for disabling report generation entirely.

Signed-off-by: Roger Peppe &lt;rogpeppe@gmail.com&gt;
diff --git a/conformance/README.md b/conformance/README.md
@@ -39,10 +39,20 @@ Lastly, run the tests:
 ./conformance.test
 ```
 
-This will produce `junit.xml` and `report.html` with the results.
-
 Note: for some registries, you may need to create `OCI_NAMESPACE` ahead of time.
 
+This will produce `junit.xml` and `report.html` in the current directory with the results. To choose an alternative directory:
+
+```
+export OCI_REPORT_DIR=/alternative/directory
+```
+
+To disable writing of the result files:
+
+```
+export OCI_REPORT_DIR=none
+```
+
 #### Testing registry workflows
 
 The tests are broken down into 4 major categories:
@@ -230,7 +240,7 @@ jobs:
       - name: Run OCI Distribution Spec conformance tests
         uses: opencontainers/distribution-spec@main
         # you can also run against a specific tag or commit instead
-        # uses: opencontainers/distribution-spec@v1.1.0 
+        # uses: opencontainers/distribution-spec@v1.1.0
         env:
           OCI_ROOT_URL: https://myreg.io
           OCI_NAMESPACE: mytestorg/mytestrepo
diff --git a/conformance/reporter.go b/conformance/reporter.go
@@ -568,6 +568,10 @@ func (reporter *HTMLReporter) afterReport(r types.SpecReport) {
 }
 
 func (reporter *HTMLReporter) endSuite(report types.Report) error {
+	if reporter.htmlReportFilename == "" {
+		// Reporting is disabled.
+		return nil
+	}
 	reporter.Report = report
 	reporter.endTime = time.Now()
 	reporter.EndTimeString = reporter.endTime.Format("Jan 2 15:04:05.000 -0700 MST")
diff --git a/conformance/setup.go b/conformance/setup.go
@@ -10,6 +10,7 @@ import (
 	"math/big"
 	mathrand "math/rand"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strconv"
 
@@ -75,6 +76,7 @@ const (
 	envVarDeleteManifestBeforeBlobs = "OCI_DELETE_MANIFEST_BEFORE_BLOBS"
 	envVarCrossmountNamespace       = "OCI_CROSSMOUNT_NAMESPACE"
 	envVarAutomaticCrossmount       = "OCI_AUTOMATIC_CROSSMOUNT"
+	envVarReportDir                 = "OCI_REPORT_DIR"
 
 	emptyLayerTestTag = "emptylayer"
 	testTagName       = "tagtest0"
@@ -510,8 +512,10 @@ func init() {
 	automaticCrossmountVal, runAutomaticCrossmountTest = os.LookupEnv(envVarAutomaticCrossmount)
 	automaticCrossmountEnabled, _ = strconv.ParseBool(automaticCrossmountVal)
 
-	reportJUnitFilename = "junit.xml"
-	reportHTMLFilename = "report.html"
+	if dir := os.Getenv(envVarReportDir); dir != "none" {
+		reportJUnitFilename = filepath.Join(dir, "junit.xml")
+		reportHTMLFilename = filepath.Join(dir, "report.html")
+	}
 	suiteDescription = "OCI Distribution Conformance Tests"
 }
 
