Merge branch 'dev' into 855-refactor-benchmark-yml

Kailai-Wang · web-flow · commit 7cb67cd9db4a · 2022-10-05T08:40:42.000+02:00
diff --git a/.github/workflows/benchmark-machine.yml b/.github/workflows/benchmark-machine.yml
@@ -0,0 +1,63 @@
+name: Benchmark machine
+
+on:
+  workflow_dispatch:
+    inputs:
+      docker_tag:
+        description: The tag of docker image to pull
+        default: 'latest'
+        required: true
+
+env:
+  INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }}  # remote AWS host to run benchmarking
+  BENCHMARK_SSH_USER: ${{ secrets.BENCHMARK_SSH_USER }}
+  BENCHMARK_SSH_KEYPATH: ${{ secrets.BENCHMARK_SSH_KEYPATH }}
+
+jobs:
+  ## run the benchmarking remotely
+  benchmark-machine:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout codes on ${{ github.ref }}
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      # TODO: maybe use GHA to start/stop remote instance
+      - name: Start remote instance
+        timeout-minutes: 10
+        id: start_instance
+        run: |
+          aws ec2 start-instances --instance-ids ${{ env.INSTANCE_ID }}
+          sleep 5
+          instance_status="aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].InstanceStatus.Status' --output text"
+          system_status="aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].SystemStatus.Status' --output text"
+          SECONDS=0
+          while : ; do
+            if [ "$(eval $instance_status)" = "ok" ] && [ "$(eval $system_status)" = "ok" ]; then
+              break
+            else
+              sleep 20
+            fi
+          done
+          echo "Remote instance reachable now after $SECONDS seconds"
+          remote_ip=`aws ec2 describe-instances --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text`
+          echo "Running instances ip address: $remote_ip"
+          echo "::set-output name=remote_ip::$remote_ip"
+
+      # exit status should propagate through ssh
+      - name: Remotely benchmark machine
+        timeout-minutes: 10
+        run: |
+          echo "Running instances ip address: ${{ steps.start_instance.outputs.remote_ip }}"
+          ssh -x -o StrictHostKeychecking=no "${{ steps.start_instance.outputs.remote_ip }}" -l ${{ env.BENCHMARK_SSH_USER }} -i ${{ env.BENCHMARK_SSH_KEYPATH }} \
+            docker pull litentry/litentry-parachain:${{ github.event.inputs.docker_tag }} && \
+            docker run --rm litentry/litentry-parachain:${{ github.event.inputs.docker_tag }} benchmark machine --chain=litmus-dev
+
+      - name: Stop remote instance
+        if: always()
+        run: |
+          aws ec2 stop-instances --instance-ids ${{ env.INSTANCE_ID }}
+          sleep 5
+          ret=`aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} | jq '.InstanceStatuses[0].InstanceState.Name'`
+          echo "Remote instance running state: $ret"
diff --git a/node/src/command.rs b/node/src/command.rs
@@ -375,6 +375,15 @@ pub fn run() -> Result<()> {
 				BenchmarkCmd::Block(cmd) => runner.sync_run(|config| {
 					construct_benchmark_partials!(config, |partials| cmd.run(partials.client))
 				}),
+				#[cfg(not(feature = "runtime-benchmarks"))]
+				BenchmarkCmd::Storage(_) =>
+					return Err(sc_cli::Error::Input(
+						"Compile with --features=runtime-benchmarks \
+						to enable storage benchmarks."
+							.into(),
+					)
+					.into()),
+				#[cfg(feature = "runtime-benchmarks")]
 				BenchmarkCmd::Storage(cmd) => runner.sync_run(|config| {
 					construct_benchmark_partials!(config, |partials| {
 						let db = partials.backend.expose_db();
