Integration of perftest CI with standardised benchmarks

.github/workflows/linux_with_bench_test.yml

@@ -0,0 +1,46 @@
+# Copyright (c) 2024 Vedant Nimje
+#
+# SPDX-License-Identifier: BSL-1.0
+# Distributed under the Boost Software License, Version 1.0. (See accompanying
+# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+name: Linux CI (Debug) with Benchmark Test
+
+on: [pull_request]
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: stellargroup/build_env:14
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Configure
+      shell: bash
+      run: |
+          cmake \
+              . \
+              -Bbuild \
+              -GNinja \
+              -DCMAKE_BUILD_TYPE=Debug \
+              -DHPX_WITH_MALLOC=system \
+              -DHPX_WITH_FETCH_ASIO=ON \
+              -DHPX_WITH_EXAMPLES=ON \
+              -DHPX_WITH_TESTS=ON \
+              -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \
+              -DHPX_WITH_CHECK_MODULE_DEPENDENCIES=On
+    - name: Build
+      shell: bash
+      run: |
+          cmake --build build/ --target tests.performance.modules.segmented_algorithms.minmax_element_performance
+    - name: Test
+      shell: bash
+      run: |
+          cd build
+          ./bin/minmax_element_performance_test
+    - name: Test with detailed output
+      shell: bash
+      run: |
+          cd build
+          ./bin/minmax_element_performance_test --hpx:detailed_bench

.github/workflows/linux_with_nanobench_test.yml

@@ -0,0 +1,48 @@
+# Copyright (c) 2024 Vedant Nimje
+#
+# SPDX-License-Identifier: BSL-1.0
+# Distributed under the Boost Software License, Version 1.0. (See accompanying
+# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+name: Linux CI (Debug) with Nanobench Benchmark Test
+
+on: [pull_request]
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: stellargroup/build_env:14
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Configure
+      shell: bash
+      run: |
+          cmake \
+              . \
+              -Bbuild \
+              -GNinja \
+              -DCMAKE_BUILD_TYPE=Debug \
+              -DHPX_WITH_MALLOC=system \
+              -DHPX_WITH_FETCH_ASIO=ON \
+              -DHPX_WITH_EXAMPLES=ON \
+              -DHPX_WITH_TESTS=ON \
+              -DHPX_WITH_NANOBENCH=ON \
+              -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \
+              -DHPX_WITH_CHECK_MODULE_DEPENDENCIES=On
+    - name: Build
+      shell: bash
+      run: |
+          cmake --build build/ --target tests.performance.modules.segmented_algorithms.minmax_element_performance
+    - name: Test
+      shell: bash
+      run: |
+          cd build
+          ./bin/minmax_element_performance_test
+
+    - name: Test with detailed output
+      shell: bash
+      run: |
+          cd build
+          ./bin/minmax_element_performance_test --hpx:detailed_bench

.jenkins/lsu-perftests/batch.sh

@@ -16,9 +16,14 @@ status_computation_and_artifacts_storage() {
 
     # Copy the testing directory for saving as an artifact
     cp -r ${build_dir}/Testing ${src_dir}/${configuration_name}-Testing
-    cp -r ${build_dir}/reports ${src_dir}/${configuration_name}-reports
+    cp -r ${build_dir}/*.json ${src_dir}/${configuration_name}-reports
 
     echo "${ctest_status}" > "jenkins-hpx-${configuration_name}-ctest-status.txt"
+
+    if [[ -s $build_dir/index.html ]]; then
+        cd $build_dir && sh $src_dir/.jenkins/lsu-perftests/comment_github.sh
+    fi
+
     exit $ctest_status
 }
 
@@ -33,7 +38,7 @@ cp -r ${src_dir}/tools/perftests_ci ${build_dir}/tools
 # Variables
 perftests_dir=${build_dir}/tools/perftests_ci
 envfile=${src_dir}/.jenkins/lsu-perftests/env-${configuration_name}.sh
-mkdir -p ${build_dir}/reports
+mkdir -p ${src_dir}/${configuration_name}-reports
 logfile=${build_dir}/reports/jenkins-hpx-${configuration_name}.log
 
 # Load python packages
@@ -48,14 +53,14 @@ plot_errors=0
 wait
 
 # Build and Run the perftests
-source ${src_dir}/.jenkins/lsu-perftests/launch_perftests.sh
+source ${src_dir}/.jenkins/lsu-perftests/env-${configuration_name}.sh
 
-# Dummy ctest to upload the html report of the perftest
-set +e
+# CTest to upload the results to CDash
 ctest \
-    --verbose \
+    -VV \
+    --output-on-failure \
     -S ${src_dir}/.jenkins/lsu-perftests/ctest.cmake \
     -DCTEST_BUILD_CONFIGURATION_NAME="${configuration_name}" \
+    -DCTEST_CONFIGURE_EXTRA_OPTIONS="${configure_extra_options}" \
     -DCTEST_SOURCE_DIRECTORY="${src_dir}" \
     -DCTEST_BINARY_DIRECTORY="${build_dir}"
-set -e

.jenkins/lsu-perftests/comment_github.sh

@@ -8,7 +8,7 @@
 
 set -eux
 
-pushd perftests-reports/reference-comparison
+# pushd perftests-reports/reference-comparison
 
 # In the order of replacement rule in sed:
 # - Remove the image as does not display in github comments (section Details in the report)
@@ -23,6 +23,4 @@ curl \
   -X POST \
   -H "Authorization: token ${GITHUB_TOKEN}" \
   https://api.github.com/repos/STEllAR-GROUP/hpx/issues/${ghprbPullId}/comments \
-  -d "{\"body\": \"<details><summary>Performance test report</summary>${report}<\/details>\"}"
-
-popd
+  -d "{\"body\": \"<details><summary>Performance test report<\/summary><table><tr><th>Test name<\/th><th>Executor<\/th><th>Percentage Diff.<\/th><th>Classification<\/th><\/tr>${report}<\/table><\/details>\"}"

.jenkins/lsu-perftests/ctest.cmake

@@ -5,7 +5,6 @@
 # Distributed under the Boost Software License, Version 1.0. (See accompanying
 # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
-# This is a dummy file to trigger the upload of the perftests reports
 cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
 
 set(CTEST_TEST_TIMEOUT 300)
@@ -20,7 +19,7 @@ set(CTEST_SUBMIT_RETRY_DELAY 60)
 
 if(NOT "$ENV{ghprbPullId}" STREQUAL "")
   set(CTEST_BUILD_NAME "$ENV{ghprbPullId}-${CTEST_BUILD_CONFIGURATION_NAME}")
-  set(CTEST_TRACK "Experimental")
+  set(CTEST_TRACK "Pull_Requests")
 else()
   set(CTEST_BUILD_NAME
       "$ENV{git_local_branch}-${CTEST_BUILD_CONFIGURATION_NAME}"
@@ -29,4 +28,99 @@ else()
 endif()
 
 ctest_start(Experimental TRACK "${CTEST_TRACK}")
+
 ctest_update()
+ctest_submit(
+  PARTS Update
+  BUILD_ID __ctest_build_id
+  RETURN_VALUE __update_result
+)
+
+set(CTEST_CONFIGURE_COMMAND "${CMAKE_COMMAND} ${CTEST_SOURCE_DIRECTORY}")
+set(CTEST_CONFIGURE_COMMAND
+    "${CTEST_CONFIGURE_COMMAND} -G${CTEST_CMAKE_GENERATOR}"
+)
+set(CTEST_CONFIGURE_COMMAND
+    "${CTEST_CONFIGURE_COMMAND} -B${CTEST_BINARY_DIRECTORY}"
+)
+set(CTEST_CONFIGURE_COMMAND
+    "${CTEST_CONFIGURE_COMMAND} -DHPX_WITH_NANOBENCH=ON"
+)
+set(CTEST_CONFIGURE_COMMAND
+    "${CTEST_CONFIGURE_COMMAND} -DHPX_WITH_PARALLEL_TESTS_BIND_NONE=ON"
+)
+set(CTEST_CONFIGURE_COMMAND
+    "${CTEST_CONFIGURE_COMMAND} ${CTEST_CONFIGURE_EXTRA_OPTIONS}"
+)
+
+ctest_configure()
+ctest_submit(
+  PARTS Configure
+  BUILD_ID __ctest_build_id
+  RETURN_VALUE __configure_result
+)
+if(NOT CTEST_BUILD_ID AND __ctest_build_id)
+  set(CTEST_BUILD_ID ${__ctest_build_id})
+endif()
+set(ctest_submission_result ${ctest_submission_result} "Configure: "
+                            ${__configure_result} "\n"
+)
+
+set(benchmarks
+  minmax_element_performance
+  small_vector_benchmark
+  future_overhead_report
+  stream_report
+  foreach_report
+  transform_reduce_scaling
+  benchmark_is_heap_until
+  benchmark_merge
+  benchmark_inplace_merge
+  benchmark_is_heap
+  benchmark_remove
+  benchmark_remove_if
+  benchmark_partial_sort
+  benchmark_partial_sort_parallel
+  benchmark_nth_element
+  benchmark_nth_element_parallel
+)
+
+foreach(benchmark ${benchmarks})
+  ctest_build(TARGET ${benchmark}_cdash_results FLAGS "-k0 -j ${CTEST_BUILD_PARALLELISM}")
+endforeach()
+
+ctest_submit(
+  PARTS Build
+  BUILD_ID __ctest_build_id
+  RETURN_VALUE __build_result
+)
+if(NOT CTEST_BUILD_ID AND __ctest_build_id)
+  set(CTEST_BUILD_ID ${__ctest_build_id})
+endif()
+set(ctest_submission_result ${ctest_submission_result} "Build: "
+                            ${__build_result} "\n"
+)
+
+ctest_test(
+  INCLUDE "_perftest$"
+  PARALLEL_LEVEL "${CTEST_TEST_PARALLELISM}"
+)
+ctest_submit(
+  PARTS Test
+  BUILD_ID __ctest_build_id
+  RETURN_VALUE __test_result
+)
+
+if(NOT CTEST_BUILD_ID AND __ctest_build_id)
+  set(CTEST_BUILD_ID ${__ctest_build_id})
+endif()
+set(ctest_submission_result ${ctest_submission_result} "Tests: "
+                            ${__test_result} "\n"
+)
+
+file(WRITE "jenkins-hpx-${CTEST_BUILD_CONFIGURATION_NAME}-cdash-build-id.txt"
+     "${CTEST_BUILD_ID}"
+)
+file(WRITE "jenkins-hpx-${CTEST_BUILD_CONFIGURATION_NAME}-cdash-submission.txt"
+     ${ctest_submission_result}
+)

.jenkins/lsu-perftests/entry.sh

@@ -53,9 +53,36 @@ status_file="jenkins-hpx-${configuration_name}-ctest-status.txt"
 
 # Comment on the PR if any failures
 if [[ $(cat ${status_file}) != 0 ]]; then
-    ./.jenkins/lsu-perftests/comment_github.sh
+    github_commit_status="failure"
+else
+    github_commit_status="success"
 fi
 
+cdash_build_id="$(cat jenkins-hpx-${configuration_name}-cdash-build-id.txt)"
+
+if [[ -z "${ghprbPullId:-}" ]]; then
+    .jenkins/common/set_github_status.sh \
+        "${GITHUB_TOKEN}" \
+        "STEllAR-GROUP/hpx" \
+        "${GIT_COMMIT}" \
+        "${github_commit_status}" \
+        "${configuration_name}" \
+        "${cdash_build_id}" \
+        "jenkins/lsu-perftests"
+else
+    # Extract just the organization and repo names "org/repo" from the full URL
+    github_commit_repo="$(echo $ghprbPullLink | sed -n 's/https:\/\/github.com\/\(.*\)\/pull\/[0-9]*/\1/p')"
+
+    # Set GitHub status with CDash url
+    .jenkins/common/set_github_status.sh \
+        "${GITHUB_TOKEN}" \
+        "${github_commit_repo}" \
+        "${ghprbActualCommit}" \
+        "${github_commit_status}" \
+        "${configuration_name}" \
+        "${cdash_build_id}" \
+        "jenkins/lsu-perftests"
+fi
 
 set -e
 exit $(cat ${status_file})

.jenkins/lsu-perftests/env-perftests.sh

@@ -16,3 +16,7 @@ module load hwloc
 module load openmpi
 
 export CXX_STD="20"
+
+configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}"
+configure_extra_options+=" -DHPX_WITH_MALLOC=system"
+configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON"