Skip to content

Commit 7ffbf11

Browse files
pfuriopfurio
committed
Merge branch 'develop' into TASK-7353
2 parents 1f50f4c + a429a6e commit 7ffbf11

23 files changed

Lines changed: 1200 additions & 66 deletions

File tree

‎.github/workflows/build-java-app-workflow.yml‎

Lines changed: 72 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,36 +10,103 @@ on:
1010
type: string
1111
required: false
1212
default: "build-folder"
13+
upload_artifact:
14+
type: boolean
15+
required: false
16+
default: true
17+
dependency_repos:
18+
# Comma-separated list of dependency repositories to clone and build before building the main project.
19+
type: string
20+
required: false
21+
default: ""
22+
needs_hadoop_preparation:
23+
type: boolean
24+
required: false
25+
default: false
26+
hadoop_flavour:
27+
type: string
28+
required: false
29+
default: ""
30+
java_commons_libs_branch:
31+
type: string
32+
required: false
33+
default: "develop"
1334
outputs:
1435
version:
1536
description: "Project version"
1637
value: ${{ jobs.build-workflow.outputs.version }}
38+
cache_key:
39+
description: "Cache key used for Maven repository"
40+
value: ${{ jobs.build-workflow.outputs.cache_key }}
1741

1842
jobs:
1943
build-workflow:
2044
name: Build Java app
2145
runs-on: ${{ vars.UBUNTU_VERSION }}
2246
outputs:
2347
version: ${{ steps.get_project_version.outputs.version }}
48+
cache_key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }}
2449
steps:
2550
- uses: actions/checkout@v4
51+
name: Checkout main repository
52+
id: "checkout-main"
53+
## This checkout pulls the code of the repository where this workflow is being used
2654
with:
2755
fetch-depth: '10'
56+
- uses: actions/checkout@v4
57+
if: ${{ inputs.dependency_repos != '' }}
58+
name: Checkout java-common-libs repository
59+
id: "checkout-java-common-libs"
60+
## This checkout pulls the code of the java-common-libs repository to get the scripts
61+
## Checkout to "java-common-libs" folder.
62+
## Filter by ".github" folder
63+
with:
64+
repository: opencb/java-common-libs
65+
ref: ${{ inputs.java_commons_libs_branch }}
66+
path: java-common-libs
67+
fetch-depth: '1'
2868
- name: Set up JDK 8
2969
uses: actions/setup-java@v4
3070
with:
3171
distribution: 'temurin'
3272
java-version: '8'
3373
cache: 'maven'
34-
- name: Install dependencies branches
74+
- name: Clone dependencies
75+
id: clone_dependencies
76+
if: ${{ inputs.dependency_repos != '' }}
77+
run: |
78+
if [ -f "java-common-libs/.github/workflows/scripts/get_same_branch.sh" ]; then
79+
chmod +x java-common-libs/.github/workflows/scripts/get_same_branch.sh
80+
export DEPENDENCIES_SHA=${{ github.sha }}
81+
java-common-libs/.github/workflows/scripts/get_same_branch.sh "${{ github.ref_name }}" "${{ inputs.dependency_repos }}"
82+
fi
83+
- name: Cache local Maven repository
84+
id: cache
85+
uses: actions/cache@v4
86+
with:
87+
path: ~/.m2/repository
88+
key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }}
89+
restore-keys: |
90+
${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-
91+
- name: Compile dependencies
92+
if: ${{ inputs.dependency_repos != '' }}
3593
run: |
36-
if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
37-
chmod +x ./.github/workflows/scripts/get_same_branch.sh
38-
./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
94+
if [ -f "java-common-libs/.github/workflows/scripts/compile_same_branch.sh" ]; then
95+
chmod +x java-common-libs/.github/workflows/scripts/compile_same_branch.sh
96+
java-common-libs/.github/workflows/scripts/compile_same_branch.sh "${{ inputs.dependency_repos }}"
3997
fi
98+
- name: Prepare Hadoop profile
99+
if: ${{ inputs.needs_hadoop_preparation == true }}
100+
run: |
101+
chmod +x ./.github/workflows/scripts/prepare_hadoop.sh
102+
./.github/workflows/scripts/prepare_hadoop.sh --hadoop-flavour "${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}"
103+
env:
104+
THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }}
105+
40106
- name: Maven Build (skip tests)
41107
run: mvn -T 2 clean install -DskipTests ${{ inputs.maven_opts }} --no-transfer-progress
42108
- uses: actions/upload-artifact@v4
109+
if: ${{ inputs.upload_artifact == true }}
43110
with:
44111
name: ${{ inputs.build_folder }}
45112
path: build
@@ -49,3 +116,4 @@ jobs:
49116
echo "version=`mvn help:evaluate -q -Dexpression=project.version -DforceStdout`" >> $GITHUB_OUTPUT
50117
- name: test-version-from-check
51118
run: echo "Project version is " ${{ steps.get_project_version.outputs.version }}
119+

‎.github/workflows/debug-token.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
name: Debug ZETTA_REPO_ACCESS_TOKEN and Permissions
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
debug-token-and-permissions:
8+
runs-on: ${{ vars.UBUNTU_VERSION }}
9+
env:
10+
ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
11+
steps:
12+
- uses: actions/checkout@v4
13+
with:
14+
fetch-depth: 1
15+
- name: Show runner and workflow context
16+
run: |
17+
echo "Runner: $(uname -a)"
18+
echo "GitHub Actor: $GITHUB_ACTOR"
19+
echo "Workflow: $GITHUB_WORKFLOW"
20+
echo "Event: $GITHUB_EVENT_NAME"
21+
echo "Repository: $GITHUB_REPOSITORY"
22+
echo "Job: $GITHUB_JOB"
23+
echo "Workspace: $GITHUB_WORKSPACE"
24+
echo "Home: $HOME"
25+
echo "Shell: $SHELL"
26+
echo "GitHub Ref: $GITHUB_REF"
27+
echo "GitHub SHA: $GITHUB_SHA"
28+
echo "GitHub Run ID: $GITHUB_RUN_ID"
29+
echo "GitHub Run Number: $GITHUB_RUN_NUMBER"
30+
31+
- name: Show GitHub Actions permissions
32+
run: |
33+
echo "Permissions JSON:"
34+
cat $GITHUB_EVENT_PATH | jq '.workflow_run?.permissions // .permissions // "No permissions found"'
35+
36+
- name: Check ZETTA_REPO_ACCESS_TOKEN presence and length
37+
run: |
38+
if [ -z "$ZETTA_REPO_ACCESS_TOKEN" ]; then
39+
echo "ZETTA_REPO_ACCESS_TOKEN is NOT set"
40+
exit 1
41+
else
42+
echo "ZETTA_REPO_ACCESS_TOKEN is set"
43+
echo "Length: ${#ZETTA_REPO_ACCESS_TOKEN}"
44+
echo "First 4 chars: ${ZETTA_REPO_ACCESS_TOKEN:0:4}"
45+
fi
46+
- id: get_ci_core_xetabase_branch
47+
name: "Get CI-CORE current branch for Xetabase from target branch"
48+
run: |
49+
chmod +x ./.github/workflows/scripts/get_opencga_enterprise_branch.sh
50+
echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}"
51+
xetabase_branch=$(./.github/workflows/scripts/get_opencga_enterprise_branch.sh "java-common-libs" "develop" "TASK-8067")
52+
echo "__CI CORE Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
53+
REPO_URI="https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git"
54+
echo "$(git ls-remote "$REPO_URI" "TASK-8067")"
55+
env:
56+
ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
57+
- id: get_xetabase_branch
58+
name: "Get JCL current branch for Xetabase from target branch"
59+
run: |
60+
chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh
61+
echo "github.event.pull_request.base.ref: develop"
62+
echo "github.event.pull_request.head.ref: TASK-8067"
63+
echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}"
64+
xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "develop" "TASK-8067")
65+
echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
66+
env:
67+
ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
68+
- name: Clone OpenCGA Enterprise branch '${{ inputs.branch }}'
69+
uses: actions/checkout@v4
70+
with:
71+
repository: zetta-genomics/opencga-enterprise
72+
ref: TASK-8067
73+
token: ${{ env.ZETTA_REPO_ACCESS_TOKEN }}
74+
path: xetbase-opencga-enterprise
75+
fetch-depth: "10"
76+
77+
- name: Try to clone the private repo with token
78+
run: |
79+
set -x
80+
git clone https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git || true
81+
82+
- name: Show authenticated user (if possible)
83+
run: |
84+
curl -H "Authorization: token $ZETTA_REPO_ACCESS_TOKEN" https://api.github.com/user
85+
86+
- name: Show environment variables (filtered)
87+
run: |
88+
env | grep -E 'GITHUB|ZETTA|RUNNER|CI'

‎.github/workflows/deploy-maven-repository-workflow.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ on:
66
maven_opts:
77
type: string
88
required: false
9+
cache_key:
10+
type: string
11+
required: false
912
secrets:
1013
MAVEN_GPG_PASSPHRASE:
1114
required: true
@@ -32,6 +35,14 @@ jobs:
3235
server-password: MAVEN_NEXUS_PASSWORD # env variable for token in deploy
3336
gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} # Value of the GPG private key to import
3437
gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase
38+
- name: Cache local Maven repository
39+
uses: actions/cache/restore@v4
40+
if: ${{ inputs.cache_key != '' }}
41+
with:
42+
path: ~/.m2/repository
43+
key: ${{ inputs.cache_key }}
44+
## Force cache hit to avoid analyzing with incomplete dependencies
45+
fail-on-cache-miss: true
3546
- name: Deploy to Maven Central repository
3647
run: mvn clean deploy -DskipTests -Pdeploy-maven ${{ inputs.maven_opts }} --no-transfer-progress
3748
env:

‎.github/workflows/develop.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,5 @@ jobs:
1414
uses: ./.github/workflows/deploy-maven-repository-workflow.yml
1515
needs: build
1616
secrets: inherit
17+
with:
18+
cache_key: ${{ needs.build.outputs.cache_key }}

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ jobs:
1313
uses: ./.github/workflows/deploy-maven-repository-workflow.yml
1414
needs: build
1515
secrets: inherit
16+
with:
17+
cache_key: ${{ needs.build.outputs.cache_key }}
1618

1719
release:
1820
uses: ./.github/workflows/release-github-workflow.yml

‎.github/workflows/scripts/compile_same_branch.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
2+
WORKSPACE=${WORKSPACE:-/home/runner/work/}
3+
4+
5+
function compile() {
6+
local REPO=$1
7+
if [ ! -d "${WORKSPACE}/$REPO" ]; then
8+
echo "Directory ${WORKSPACE}/$REPO does not exist. Skip compile"
9+
return 0;
10+
fi
11+
echo "::group::Compiling '$REPO' project from branch $BRANCH_NAME"
12+
cd "${WORKSPACE}/$REPO" || exit 2
13+
mvn clean install -DskipTests --no-transfer-progress
14+
echo "::endgroup::"
15+
}
16+
17+
## Comma separated list of repos to compile
18+
REPOS=$1
19+
20+
IFS=',' read -ra REPO_ARRAY <<<"$REPOS"
21+
for REPO in "${REPO_ARRAY[@]}"; do
22+
compile "$REPO"
23+
done

‎.github/workflows/scripts/get-xetabase-branch.sh‎

100644100755
Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,13 @@ get_xetabase_branch() {
1010

1111
# If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it
1212
if [[ $current_branch == TASK* ]]; then
13-
if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$current_branch" )" ] ; then
13+
REPO_URI=
14+
if [ -z "$ZETTA_REPO_ACCESS_TOKEN" ]; then
15+
REPO_URI="git@github.com:zetta-genomics/opencga-enterprise.git"
16+
else
17+
REPO_URI="https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git"
18+
fi
19+
if [ "$(git ls-remote "$REPO_URI" "$current_branch" )" ] ; then
1420
echo "$current_branch";
1521
return 0;
1622
fi

‎.github/workflows/scripts/get_opencga_enterprise_branch.sh‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#!/usr/bin/env bash
2+
# get_opencga_enterprise_branch.sh
3+
#
4+
# This script computes the correct opencga-enterprise branch to use for testing a PR in any Xetabase component repo.
5+
# It unifies the logic previously duplicated in java-common-libs, biodata, and opencga.
6+
#
7+
# Inputs:
8+
# 1. project (artifactId from pom.xml)
9+
# 2. base_ref (target branch of the PR)
10+
# 3. head_ref (source branch of the PR)
11+
#
12+
# Output:
13+
# Prints the resolved opencga-enterprise branch to stdout.
14+
# Exits with non-zero and error message if it cannot resolve a branch.
15+
16+
set -euo pipefail
17+
18+
project="$1"
19+
base_ref="$2"
20+
head_ref="$3"
21+
REPO_URI="https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git"
22+
23+
# 1. If the branch begins with 'TASK' and exists in the opencga-enterprise repository, return it
24+
if [[ $head_ref == TASK* ]]; then
25+
if [ "$(git ls-remote "$REPO_URI" "$head_ref")" ] ; then
26+
echo "$head_ref";
27+
exit 0
28+
fi
29+
fi
30+
31+
# 2. If the base_ref is 'develop', always map to develop
32+
if [[ "$base_ref" == "develop" ]]; then
33+
echo "develop"
34+
exit 0
35+
fi
36+
37+
# 3. release-* branch logic
38+
if [[ "$base_ref" =~ ^release-([0-9]+)\. ]]; then
39+
major="${BASH_REMATCH[1]}"
40+
# Project-specific offset for release branch mapping
41+
case "$project" in
42+
java-common-libs)
43+
offset=3
44+
;;
45+
biodata|opencga)
46+
offset=1
47+
;;
48+
opencga-enterprise)
49+
# If the project is opencga-enterprise, use the branch as-is
50+
echo "$base_ref"
51+
exit 0
52+
;;
53+
*)
54+
echo "ERROR: Unknown project '$project' for release branch mapping" >&2
55+
exit 1
56+
;;
57+
esac
58+
new_major=$((major - offset))
59+
if (( new_major < 1 )); then
60+
echo "ERROR: Computed release branch version < 1 for $project (base_ref: $base_ref, offset: $offset)" >&2
61+
exit 1
62+
fi
63+
# Extract the rest of the version (minor and patch) from base_ref
64+
rest_version=$(echo "$base_ref" | sed -E "s/^release-[0-9]+(\..*)/\1/")
65+
target_branch="release-${new_major}${rest_version}"
66+
# Check if the exact branch exists (fix: do not escape quotes)
67+
if [ "$(git ls-remote --heads "$REPO_URI" "$target_branch")" ]; then
68+
echo "$target_branch"
69+
exit 0
70+
else
71+
echo "ERROR: No matching release branch '$target_branch' found in opencga-enterprise for $project (base_ref: $base_ref, offset: $offset)" >&2
72+
exit 1
73+
fi
74+
fi
75+
76+
# 4. Fallback: fail with clear error
77+
echo "ERROR: Could not resolve opencga-enterprise branch for project '$project' (base_ref: $base_ref, head_ref: $head_ref)" >&2
78+
exit 1
79+

0 commit comments

Comments
 (0)