Skip to content

Commit 6d736a9

Browse files
author
Surukanti
committed
Add comprehensive GitHub Actions CI/CD workflows
Workflows added: - CI: Tests, linting, security scanning (flake8, black, isort, bandit) - Docker: Build and push images to Docker Hub & GitHub Container Registry - Quality: Code analysis, complexity, dependency checks, SonarCloud - Docs: Auto-generate and deploy documentation to GitHub Pages - Release: Create releases and publish to PyPI Configuration files: - setup.py: PyPI package configuration - pyproject.toml: Project metadata and tool config - sonar-project.properties: SonarCloud analysis config - .pre-commit-config.yaml: Local git hooks for code quality Features: - Automated testing on every push and PR - Docker image scanning with Trivy - Code coverage reports with Codecov - Semantic versioning support - GitHub Pages documentation deployment - PyPI publishing automation
1 parent d8f5804 commit 6d736a9

10 files changed

Lines changed: 891 additions & 0 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
name: CI - Tests & Linting
2+
3+
on:
4+
push:
5+
branches: [ main, develop ]
6+
pull_request:
7+
branches: [ main, develop ]
8+
9+
jobs:
10+
test:
11+
runs-on: ubuntu-latest
12+
13+
services:
14+
redis:
15+
image: redis:7-alpine
16+
options: >-
17+
--health-cmd "redis-cli ping"
18+
--health-interval 10s
19+
--health-timeout 5s
20+
--health-retries 5
21+
ports:
22+
- 6379:6379
23+
24+
steps:
25+
- uses: actions/checkout@v3
26+
27+
- name: Set up Python 3.11
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: '3.11'
31+
cache: 'pip'
32+
33+
- name: Install dependencies
34+
run: |
35+
python -m pip install --upgrade pip
36+
pip install -r requirements.txt
37+
pip install pytest pytest-cov flake8 black isort
38+
39+
- name: Lint with flake8
40+
run: |
41+
# Stop the build if there are Python syntax errors or undefined names
42+
flake8 src/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
43+
# exit-zero treats all errors as warnings
44+
flake8 src/ tests/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
45+
46+
- name: Check code formatting with black
47+
run: black --check src/ tests/ || true
48+
49+
- name: Check import sorting with isort
50+
run: isort --check-only src/ tests/ || true
51+
52+
- name: Run stress test
53+
env:
54+
REDIS_HOST: localhost
55+
REDIS_PORT: 6379
56+
run: |
57+
# Start services in background
58+
python src/gateway/server.py &
59+
GATEWAY_PID=$!
60+
sleep 2
61+
62+
# Run stress test
63+
python tests/stress_test.py
64+
65+
# Cleanup
66+
kill $GATEWAY_PID || true
67+
timeout-minutes: 5
68+
69+
- name: Generate coverage report
70+
env:
71+
REDIS_HOST: localhost
72+
REDIS_PORT: 6379
73+
run: |
74+
pytest tests/ --cov=src/ --cov-report=xml --cov-report=html || true
75+
continue-on-error: true
76+
77+
- name: Upload coverage to Codecov
78+
uses: codecov/codecov-action@v3
79+
with:
80+
file: ./coverage.xml
81+
fail_ci_if_error: false
82+
83+
security-scan:
84+
runs-on: ubuntu-latest
85+
86+
steps:
87+
- uses: actions/checkout@v3
88+
89+
- name: Set up Python 3.11
90+
uses: actions/setup-python@v4
91+
with:
92+
python-version: '3.11'
93+
94+
- name: Install dependencies
95+
run: |
96+
python -m pip install --upgrade pip
97+
pip install bandit safety
98+
99+
- name: Security check with bandit
100+
run: bandit -r src/ -ll || true
101+
102+
- name: Check dependencies with safety
103+
run: |
104+
pip install -r requirements.txt
105+
safety check || true

‎.github/workflows/docker.yml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
name: Docker Build & Push
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
tags: [ 'v*' ]
7+
pull_request:
8+
branches: [ main ]
9+
10+
jobs:
11+
build:
12+
runs-on: ubuntu-latest
13+
permissions:
14+
contents: read
15+
packages: write
16+
17+
strategy:
18+
matrix:
19+
include:
20+
- dockerfile: Dockerfile.gateway
21+
image_name: distributed-task-queue-gateway
22+
context: .
23+
- dockerfile: Dockerfile.worker
24+
image_name: distributed-task-queue-worker
25+
context: .
26+
- dockerfile: Dockerfile.dashboard
27+
image_name: distributed-task-queue-dashboard
28+
context: .
29+
30+
steps:
31+
- uses: actions/checkout@v3
32+
33+
- name: Set up Docker Buildx
34+
uses: docker/setup-buildx-action@v2
35+
36+
- name: Log in to Docker Hub
37+
if: github.event_name != 'pull_request'
38+
uses: docker/login-action@v2
39+
with:
40+
username: ${{ secrets.DOCKER_USERNAME }}
41+
password: ${{ secrets.DOCKER_PASSWORD }}
42+
43+
- name: Log in to GitHub Container Registry
44+
if: github.event_name != 'pull_request'
45+
uses: docker/login-action@v2
46+
with:
47+
registry: ghcr.io
48+
username: ${{ github.actor }}
49+
password: ${{ secrets.GITHUB_TOKEN }}
50+
51+
- name: Extract metadata
52+
id: meta
53+
uses: docker/metadata-action@v4
54+
with:
55+
images: |
56+
${{ secrets.DOCKER_USERNAME }}/${{ matrix.image_name }}
57+
ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}
58+
tags: |
59+
type=ref,event=branch
60+
type=semver,pattern={{version}}
61+
type=semver,pattern={{major}}.{{minor}}
62+
type=sha
63+
64+
- name: Build and push Docker image
65+
uses: docker/build-push-action@v4
66+
with:
67+
context: ${{ matrix.context }}
68+
file: ${{ matrix.dockerfile }}
69+
push: ${{ github.event_name != 'pull_request' }}
70+
tags: ${{ steps.meta.outputs.tags }}
71+
labels: ${{ steps.meta.outputs.labels }}
72+
cache-from: type=gha
73+
cache-to: type=gha,mode=max
74+
75+
docker-scan:
76+
runs-on: ubuntu-latest
77+
if: github.event_name == 'push'
78+
79+
strategy:
80+
matrix:
81+
dockerfile:
82+
- Dockerfile.gateway
83+
- Dockerfile.worker
84+
- Dockerfile.dashboard
85+
86+
steps:
87+
- uses: actions/checkout@v3
88+
89+
- name: Set up Docker Buildx
90+
uses: docker/setup-buildx-action@v2
91+
92+
- name: Build Docker image for scanning
93+
uses: docker/build-push-action@v4
94+
with:
95+
context: .
96+
file: ${{ matrix.dockerfile }}
97+
load: true
98+
tags: image-to-scan:latest
99+
100+
- name: Run Trivy vulnerability scan
101+
uses: aquasecurity/trivy-action@master
102+
with:
103+
image-ref: 'image-to-scan:latest'
104+
format: 'sarif'
105+
output: 'trivy-results.sarif'
106+
107+
- name: Upload Trivy results to GitHub Security tab
108+
uses: github/codeql-action/upload-sarif@v2
109+
with:
110+
sarif_file: 'trivy-results.sarif'

‎.github/workflows/docs.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: Deploy Documentation
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build-docs:
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- uses: actions/checkout@v3
15+
16+
- name: Set up Python
17+
uses: actions/setup-python@v4
18+
with:
19+
python-version: '3.11'
20+
21+
- name: Install dependencies
22+
run: |
23+
python -m pip install --upgrade pip
24+
pip install mkdocs mkdocs-material pymdown-extensions
25+
26+
- name: Build documentation
27+
run: |
28+
mkdir -p docs/source
29+
# Copy documentation files
30+
cp README.md docs/source/index.md
31+
cp FEATURES_SUMMARY.md docs/source/features.md
32+
cp ADVANCED_FEATURES.md docs/source/advanced.md
33+
cp QUICK_START_FEATURES.md docs/source/quickstart.md
34+
cp COMMAND_REFERENCE.md docs/source/commands.md
35+
36+
# Create mkdocs.yml if it doesn't exist
37+
if [ ! -f mkdocs.yml ]; then
38+
cat > mkdocs.yml << 'EOF'
39+
site_name: Distributed Task Queue
40+
theme:
41+
name: material
42+
palette:
43+
scheme: slate
44+
primary: purple
45+
accent: pink
46+
nav:
47+
- Home: index.md
48+
- Features: features.md
49+
- Quick Start: quickstart.md
50+
- Advanced: advanced.md
51+
- Commands: commands.md
52+
EOF
53+
fi
54+
55+
mkdocs build
56+
57+
- name: Deploy to GitHub Pages
58+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
59+
uses: peaceiris/actions-gh-pages@v3
60+
with:
61+
github_token: ${{ secrets.GITHUB_TOKEN }}
62+
publish_dir: ./site
63+
cname: distributed-task-queue.example.com

‎.github/workflows/quality.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
name: Code Quality & Analysis
2+
3+
on:
4+
push:
5+
branches: [ main, develop ]
6+
pull_request:
7+
branches: [ main, develop ]
8+
9+
jobs:
10+
quality:
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- uses: actions/checkout@v3
15+
with:
16+
fetch-depth: 0 # Full history for better analysis
17+
18+
- name: Set up Python 3.11
19+
uses: actions/setup-python@v4
20+
with:
21+
python-version: '3.11'
22+
cache: 'pip'
23+
24+
- name: Install dependencies
25+
run: |
26+
python -m pip install --upgrade pip
27+
pip install -r requirements.txt
28+
pip install pylint mypy radon
29+
30+
- name: Analyze code complexity with Radon
31+
run: |
32+
radon cc src/ -a --show-closures || true
33+
radon mi src/ --min B || true
34+
35+
- name: Type check with mypy
36+
run: mypy src/ --ignore-missing-imports || true
37+
38+
- name: Lint with pylint
39+
run: pylint src/ --exit-zero --disable=all --enable=E,F || true
40+
41+
- name: Check docstring coverage
42+
run: |
43+
pip install interrogate
44+
interrogate -vv src/ --fail-under=50 --ignore-init-method || true
45+
46+
dependency-check:
47+
runs-on: ubuntu-latest
48+
49+
steps:
50+
- uses: actions/checkout@v3
51+
52+
- name: Check for outdated dependencies
53+
run: |
54+
python -m pip install --upgrade pip
55+
pip install pip-audit
56+
pip install -r requirements.txt
57+
pip-audit || true
58+
59+
- name: Dependency check with OWASP
60+
uses: dependency-check/Dependency-Check_Action@main
61+
with:
62+
project: 'distributed-task-queue-python'
63+
path: '.'
64+
format: 'JSON'
65+
continue-on-error: true
66+
67+
sonarcloud:
68+
runs-on: ubuntu-latest
69+
70+
steps:
71+
- uses: actions/checkout@v3
72+
with:
73+
fetch-depth: 0
74+
75+
- name: SonarCloud Scan
76+
uses: SonarSource/sonarcloud-github-action@master
77+
env:
78+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
79+
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
80+
continue-on-error: true

0 commit comments

Comments
 (0)