flytekit/.github/workflows/pythonbuild.yml at master · ProjectAussie/flytekit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
name: Build

on:
  push:
    branches:
      - master
  pull_request:

env:
  FLYTE_SDK_LOGGING_LEVEL: 10 # debug

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
        python-version: ["3.7", "3.8", "3.9", "3.10"]
        spark-version-suffix: ["", "-spark2"]
        exclude:
          - python-version: 3.8
            spark-version-suffix: "-spark2"
          - python-version: 3.9
            spark-version-suffix: "-spark2"
          - python-version: 3.10
            spark-version-suffix: "-spark2"
          # Ignore this test because we failed to install docker-py
          # docker-py will install pywin32==227, whereas pywin only added support for python 3.10 in version 301.
          # For more detail, see https://github.com/flyteorg/flytekit/pull/856#issuecomment-1067152855
          - python-version: 3.10
            os: windows-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements files
          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
      - name: Install dependencies
        run: |
          make setup${{ matrix.spark-version-suffix }}
          pip freeze
      - name: Test with coverage
        run: |
          pytest tests/flytekit/unit -m "not sandbox_test" --cov=./ --cov-report=xml
      - name: Codecov
        uses: codecov/codecov-action@v3.1.0
        with:
          fail_ci_if_error: true # optional (default = false)

  build-plugins:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
        plugin-names:
          # Please maintain an alphabetical order in the following list
          - flytekit-aws-athena
          - flytekit-aws-batch
          - flytekit-aws-sagemaker
          - flytekit-bigquery
          - flytekit-data-fsspec
          - flytekit-dbt
          - flytekit-deck-standard
          - flytekit-dolt
          - flytekit-greatexpectations
          - flytekit-hive
          - flytekit-k8s-pod
          - flytekit-kf-mpi
          - flytekit-kf-pytorch
          - flytekit-kf-tensorflow
          - flytekit-modin
          - flytekit-onnx-pytorch
          - flytekit-onnx-scikitlearn
          - flytekit-onnx-tensorflow
          - flytekit-pandera
          - flytekit-papermill
          - flytekit-polars
          - flytekit-ray
          - flytekit-snowflake
          - flytekit-spark
          - flytekit-sqlalchemy
          - flytekit-whylogs
        exclude:
          # flytekit-modin depends on ray which does not have a 3.10 wheel yet.
          # Issue tracked in https://github.com/ray-project/ray/issues/19116.
          - python-version: 3.10
            plugin-names: "flytekit-modin"
          # Great-expectations does not support python 3.10 yet
          # https://github.com/great-expectations/great_expectations/blob/develop/setup.py#L87-L89
          - python-version: 3.10
            plugin-names: "flytekit-greatexpectations"
          # onnxruntime does not support python 3.10 yet
          # https://github.com/microsoft/onnxruntime/issues/9782
          - python-version: 3.10
            plugin-names: "flytekit-onnx-pytorch"
          - python-version: 3.10
            plugin-names: "flytekit-onnx-scikitlearn"
          - python-version: 3.10
            plugin-names: "flytekit-onnx-tensorflow"
          # whylogs-sketching library does not have a 3.10 build yet
          # Issue tracked: https://github.com/whylabs/whylogs/issues/697
          - python-version: 3.10
            plugin-names: "flytekit-whylogs"
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements files
          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }}
      - name: Install dependencies
        run: |
          make setup
          cd plugins/${{ matrix.plugin-names }}
          pip install -r requirements.txt
          if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
          pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
          pip freeze
      - name: Test with coverage
        run: |
          cd plugins/${{ matrix.plugin-names }}
          coverage run -m pytest tests
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Fetch the code
        uses: actions/checkout@v2
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip==21.2.4
          pip install -r dev-requirements.txt
      - name: Lint
        run: |
          make lint
      - name: ShellCheck
        uses: ludeeus/action-shellcheck@master
        with:
          ignore_paths: boilerplate

  docs:
    runs-on: ubuntu-latest
    steps:
      - name: Fetch the code
        uses: actions/checkout@v2
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip==21.2.4 setuptools wheel
          pip install -r doc-requirements.txt
      - name: Build the documentation
        run: make -C docs html