Automate conformance testing on PR using zot

jdolitsky · jdolitsky · commit bea9b6fe36e5 · 2023-04-04T18:25:53.000-05:00
Signed-off-by: Josh Dolitsky &lt;josh@dolit.ski&gt;
diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml
@@ -23,8 +23,32 @@ jobs:
           go-version: ${{ matrix.go }}
 
       - name: run tests
+        id: tests
         run: |
           export PATH="$(go env GOPATH)/bin:${PATH}"
           make install.tools
           make .gitvalidation
           make docs conformance
+
+          set +e
+          make conformance-ci
+          CONFORMANCE_RC="$?"
+          set -e
+          if [[ -f report.html ]]; then
+            echo "Found report.html."
+            echo "has-report=true" >> $GITHUB_OUTPUT
+          fi
+
+          # TODO: Fail with CONFORMANCE_RC if non-zero
+          # (conformance not yet passing for this registry)
+          echo "Conformance return code: ${CONFORMANCE_RC}"
+          exit 0
+
+      - name: Upload OCI conformance results as build artifact
+        if: always() && steps.tests.outputs.has-report == 'true'
+        uses: actions/upload-artifact@v3
+        with:
+          name: oci-conformance-results-${{ matrix.go }}
+          path: |
+            ./report.html
+            ./junit.xml
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -23,8 +23,32 @@ jobs:
           go-version: ${{ matrix.go }}
 
       - name: run tests
+        id: tests
         run: |
           export PATH="$(go env GOPATH)/bin:${PATH}"
           make install.tools
           make .gitvalidation
           make docs conformance
+
+          set +e
+          make conformance-ci
+          CONFORMANCE_RC="$?"
+          set -e
+          if [[ -f report.html ]]; then
+            echo "Found report.html."
+            echo "has-report=true" >> $GITHUB_OUTPUT
+          fi
+
+          # TODO: Fail with CONFORMANCE_RC if non-zero
+          # (conformance not yet passing for this registry)
+          echo "Conformance return code: ${CONFORMANCE_RC}"
+          exit 0
+
+      - name: Upload OCI conformance results as build artifact
+        if: always() && steps.tests.outputs.has-report == 'true'
+        uses: actions/upload-artifact@v3
+        with:
+          name: oci-conformance-results-${{ matrix.go }}
+          path: |
+            ./report.html
+            ./junit.xml
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,5 @@ header.html
 tags
 go.mod
 go.sum
+junit.xml
+report.html
diff --git a/Makefile b/Makefile
@@ -91,6 +91,24 @@ conformance-test:
 
 conformance-binary: $(OUTPUT_DIRNAME)/conformance.test
 
+TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.0.0-rc3
+conformance-ci:
+	docker rm -f oci-conformance && \
+		echo '{"distSpecVersion":"1.1.0-dev","storage":{"rootDirectory":"/tmp/zot","gc":false,"dedupe":false},"http":{"address":"0.0.0.0","port":"5000"},"log":{"level":"debug"}}' > $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json
+		docker run -d \
+			-v $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json:/etc/zot/config.json \
+			--name=oci-conformance \
+			-p 5000:5000 \
+			$(TEST_REGISTRY_CONTAINER) && \
+		export OCI_ROOT_URL="http://localhost:5000" && \
+		export OCI_NAMESPACE="myorg/myrepo" && \
+		export OCI_TEST_PULL=1 && \
+		export OCI_TEST_PUSH=1 && \
+		export OCI_TEST_CONTENT_DISCOVERY=1 && \
+		export OCI_TEST_CONTENT_MANAGEMENT=1 && \
+		sleep 5 && \
+		$(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test
+
 $(OUTPUT_DIRNAME)/conformance.test:
 	cd conformance && \
 		CGO_ENABLED=0 go test -c -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test \
