Containerize tests for katib-conformance (#2146)

.github/workflows/publish-conformance-images.yaml

@@ -0,0 +1,24 @@
+name: Publish Katib Conformance Test Images
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  core:
+    name: Publish Image
+    uses: ./.github/workflows/build-and-publish-images.yaml
+    with:
+      component-name: ${{ matrix.component-name }}
+      platforms: linux/amd64,linux/arm64
+      dockerfile: ${{ matrix.dockerfile }}
+    secrets:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - component-name: katib-conformance
+            dockerfile: Dockerfile.conformance

Dockerfile.conformance

@@ -0,0 +1,31 @@
+# Copyright 2023 The Kubeflow Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for building the source code of conformance tests
+FROM python:3.10-slim
+
+WORKDIR /kubeflow/katib
+
+COPY sdk/ /kubeflow/katib/sdk/
+COPY examples/ /kubeflow/katib/examples/
+COPY test/ /kubeflow/katib/test/
+
+COPY conformance/run.sh .
+
+# Add test script.
+RUN chmod +x run.sh
+
+RUN pip install --prefer-binary -e sdk/python/v1beta1
+
+ENTRYPOINT [ "./run.sh" ]

conformance/run.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# Run conformance test and generate test report.
+python test/e2e/v1beta1/scripts/gh-actions/run-e2e-experiment.py --experiment-path examples/v1beta1/hp-tuning/random.yaml --namespace kf-conformance \
+--trial-pod-annotations '{"sidecar.istio.io/inject": "false"}' | tee /tmp/katib-conformance.log
+
+
+# Create the done file.
+touch /tmp/katib-conformance.done
+echo "Done..."
+
+# Keep the container running so the test logs can be downloaded.
+while true; do sleep 10000; done