BitBLAS/.github/workflows/benchmark.yml at main · microsoft/BitBLAS · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
name: Benchmark

on:
  issue_comment:
    types: created

jobs:
  benchmark_base:
    # On pull requests and if the comment starts with `/run-benchmark`
    if: github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/run-benchmark')
    runs-on: self-hosted

    steps:
      - name: Checkout code
        uses: actions/checkout@v2
        with:
          ref: main

      - name: Get base branch commit ID
        id: get_base_commit
        run: echo "BASE_COMMIT_ID=$(git rev-parse HEAD)" > base_commit_id.txt

      - name: Upload base commit ID
        uses: actions/upload-artifact@v4
        with:
          name: base-commit-id
          path: base_commit_id.txt

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.9'

      - name: Create virtual environment
        run: python -m venv bitblas_benchmark

      - name: Activate virtual environment and install dependencies
        run: |
          source bitblas_benchmark/bin/activate
          python -m pip install --upgrade pip
          if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi

      - name: Install project in wheel mode
        run: |
          source bitblas_benchmark/bin/activate
          python -m pip install .

      - name: Matmul Benchmark
        run: |
          source bitblas_benchmark/bin/activate
          cd benchmark/operators
          python ./benchmark_ops_matmul.py

  benchmark_head:
    # On pull requests and if the comment starts with `/run-benchmark`
    if: github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/run-benchmark')
    runs-on: self-hosted
    needs: [benchmark_base]

    steps:
      - name: Checkout PR branch code
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref }}

      - name: Get PR branch commit ID
        id: get_pr_commit
        run: echo "PR_COMMIT_ID=$(git rev-parse HEAD)" > pr_commit_id.txt

      - name: Upload PR commit ID
        uses: actions/upload-artifact@v4
        with:
          name: pr-commit-id
          path: pr_commit_id.txt

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.9'

      - name: Create virtual environment
        run: python -m venv bitblas_benchmark

      - name: Activate virtual environment and install dependencies
        run: |
          source bitblas_benchmark/bin/activate
          python -m pip install --upgrade pip
          if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi

      - name: Install project in wheel mode
        run: |
          source bitblas_benchmark/bin/activate
          python -m pip install .

      - name: Matmul Benchmark
        run: |
          source bitblas_benchmark/bin/activate
          cd benchmark/operators
          python ./benchmark_ops_matmul.py

  benchmark_compare:
    if: github.event.issue.pull_request != null && contains(github.event.comment.body, '/run-benchmark')
    needs: [benchmark_base, benchmark_head]
    runs-on: self-hosted

    steps:
      - name: Download commit IDs
        uses: actions/download-artifact@v4.1.7
        with:
          name: base-commit-id
          path: .

      - name: Download PR commit ID
        uses: actions/download-artifact@v4.1.7
        with:
          name: pr-commit-id
          path: .

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.9'

      - name: Create virtual environment
        run: python -m venv bitblas_benchmark

      - name: Activate virtual environment and install dependencies
        run: |
          source bitblas_benchmark/bin/activate
          python -m pip install --upgrade pip
          if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi

      - name: Compare benchmark results
        run: |
          source bitblas_benchmark/bin/activate
          cd benchmark/operators
          python ./compare_benchmark.py --base $(cat base_commit_id.txt) --head $(cat pr_commit_id.txt) 2>&1 | tee compare_results.txt

      - name: Authenticate GitHub CLI
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token

      - name: Post benchmark results
        run: |
          cat compare_results.txt
          gh pr comment ${{ github.event.issue.number }} --body "$(cat compare_results.txt)"