From e313fee149fadedc26849cc9941039214388a330 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Thu, 21 Dec 2023 14:13:47 -0500 Subject: [PATCH 01/51] feat: new client workflow --- .github/workflows/generate_new_client.yaml | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/generate_new_client.yaml diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml new file mode 100644 index 000000000000..1a7133d2ebae --- /dev/null +++ b/.github/workflows/generate_new_client.yaml @@ -0,0 +1,23 @@ +name: Generate new GAPIC client library +on: + workflow_dispatch: + +jobs: + generate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + cache: 'pip' # caching pip dependencies + - name: Get current week within the year + id: date + run: echo "::set-output name=week_of_year::$(date +'%W' --utc)" + - uses: actions/setup-java@v3 + with: + distribution: zulu + java-version: 11 + - run: java -version + - run: echo 'a message' + From 181f8025341575d76881fc9d085e77a3b8374a13 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 20:39:04 +0000 Subject: [PATCH 02/51] define inputs to workflow --- .github/workflows/generate_new_client.yaml | 117 +++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 1a7133d2ebae..558a70bee5f9 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -1,6 +1,114 @@ name: Generate new GAPIC client library on: workflow_dispatch: + inputs: + api_shortname: + required: true + type: string + description: Name for the new directory name and (default) artifact name + name_pretty: + required: true + type: string + description: The human-friendly name that appears in README.md + proto_path: + required: true + type: string + description: | + Path to proto file from the root of the googleapis repository to the + directory that contains the proto files (without the version). + For example, to generate the library for 'google/maps/routing/v2', + then you specify this value as 'google/maps/routing' + product_docs: + required: true + type: string + description: Documentation URL that appears in README.md + rest_docs: + required: false + type: string + description: | + If it exists, link to the REST Documentation for a service + rpc_docs: + required: false + type: string + description: | + If it exists, link to the RPC Documentation for a service + api_description: + required: true + description: Description that appears in README.md + release_level: + required: false + type: choice + default: preview + options: + - stable + - preview + description: | + A label that appears in repo-metadata.json. + The first library generation is always 'preview'. + transport: + required: false + type: choice + default: grpc + options: + - grpc + - http + - both + description: A label that appears in repo-metadata.json + language: + required: false + type: string + default: java + distribution_name: + required: false + type: string + description: | + Maven coordinates of the generated library. By default it's + "com.google.cloud:google-cloud-" + api_id: + required: false + type: string + description: | + The value of the apiid parameter used in README.md It has link to + "https://console.cloud.google.com/flows/enableapi?apiid=" + requires_billling: + required: false + type: boolean + default: true + description: | + Based on this value, README.md explains whether billing setup is + "needed or not." + destination_name: + required: false + type: string + default: null + description: | + The directory name of the new library. By default it's + "java-" + cloud_api: + required: false + type: boolean + default: true + description: | + If true, the artifact ID of the library is 'google-cloud-' + otherwise 'google-' + group_id: + required: false + type: string + default: com.google.cloud + description: | + The group ID of the artifact when distribution name is not set + owlbot_image: + required: false + type: string + default: gcr.io/cloud-devrel-public-resources/owlbot-java + description: The owlbot container image used in OwlBot.yaml + library_type: + required: false + type: string + default: GAPIC_AUTO + description: | + A label that appear in repo-metadata.json to tell how the library is + maintained or generated jobs: generate: @@ -20,4 +128,13 @@ jobs: java-version: 11 - run: java -version - run: echo 'a message' + - name: Generate + run: | + python generation/new_client/new-client.py generate \ + --api_shortname=${API_SHORT_NAME} \ + --proto-path=${PROTO_PATH} \ + --name-pretty="${API_PRETTY_NAME}" \ + --product-docs="https://github.com/googleapis/google-cloud-java/tree/main/java-${API_SHORT_NAME}" \ + --api-description="${API_PRETTY_NAME} Client lets you use ${API_PRETTY_NAME} API." \ + --googleapis-gen-url=https://yoshi-approver:${{ secrets.YOSHI_APPROVER_PRIVATE_TOKEN }}@github.com/googleapis/googleapis-gen.git From aad104039803ea3775d6ca2023bfcbbe74c7a74a Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 21:33:13 +0000 Subject: [PATCH 03/51] initial new library script --- .github/workflows/generate_new_client.yaml | 70 ++++++++++++++----- .../workflows/verify-client-generation.yaml | 2 +- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 558a70bee5f9..eb4182432d3b 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -38,7 +38,6 @@ on: release_level: required: false type: choice - default: preview options: - stable - preview @@ -48,7 +47,6 @@ on: transport: required: false type: choice - default: grpc options: - grpc - http @@ -62,7 +60,7 @@ on: required: false type: string description: | - Maven coordinates of the generated library. By default it's + Maven coordinates of the generated library. By default it's "com.google.cloud:google-cloud-" api_id: required: false @@ -73,39 +71,33 @@ on: requires_billling: required: false type: boolean - default: true description: | Based on this value, README.md explains whether billing setup is "needed or not." destination_name: required: false type: string - default: null description: | The directory name of the new library. By default it's "java-" cloud_api: required: false type: boolean - default: true description: | If true, the artifact ID of the library is 'google-cloud-' otherwise 'google-' group_id: required: false type: string - default: com.google.cloud description: | The group ID of the artifact when distribution name is not set owlbot_image: required: false type: string - default: gcr.io/cloud-devrel-public-resources/owlbot-java description: The owlbot container image used in OwlBot.yaml library_type: required: false type: string - default: GAPIC_AUTO description: | A label that appear in repo-metadata.json to tell how the library is maintained or generated @@ -127,14 +119,60 @@ jobs: distribution: zulu java-version: 11 - run: java -version + - uses: actions/cache@v3 + id: mvn-cache + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }} + - name: Install new-client.py dependencies + run: pip install --require-hashes -r generation/new_client/requirements.txt - run: echo 'a message' - name: Generate run: | - python generation/new_client/new-client.py generate \ - --api_shortname=${API_SHORT_NAME} \ - --proto-path=${PROTO_PATH} \ - --name-pretty="${API_PRETTY_NAME}" \ - --product-docs="https://github.com/googleapis/google-cloud-java/tree/main/java-${API_SHORT_NAME}" \ - --api-description="${API_PRETTY_NAME} Client lets you use ${API_PRETTY_NAME} API." \ - --googleapis-gen-url=https://yoshi-approver:${{ secrets.YOSHI_APPROVER_PRIVATE_TOKEN }}@github.com/googleapis/googleapis-gen.git + arguments="--api_shortname=${API_SHORTNAME} \ + --proto_path=${PROTO_PATH} \ + --name-pretty=${NAME_PRETTY} \ + --product-docs=${PRODUCT_DOCS} \ + --api_description=${API_DESCRIPTION} | + --googleapis-gen-url=${GOOGLEAPIS_GEN_URL}" + + # helper function that appends a python argument only if specified in the GH action inputs + append_argument() { + py_arg=$1 + # env vars look exactly like new-client arguments but uppercase + underscores + env_name=$(echo "${py_arg}" | sed 's/-/_/g' | sed -e 's/\([a-z]\)/\U\1/g') + if [[ -n "${!env_name}" ]]; then + # $(echo) is redundant but it works around a syntax highlighting problem in vim + arguments=$(echo "${arguments} --${py_arg}=${!env_name}") + fi + } + + declare -a optional_args=('release-level' 'transport' 'distribution-name' 'api-id' + 'requires-billing' 'destination-name' 'cloud-api' 'group-id' 'owlbot-image' + 'library-type' 'rest-docs' 'rpc-docs') + + for python_argument in "${optional_args[@]}"; do + append_argument "${python_argument}" + done + echo "${arguments}" | xargs -0 python generation/new_client/new-client.py generate + env: + GOOGLEAPIS_GEN_URL: https://yoshi-approver:${{ secrets.YOSHI_APPROVER_PRIVATE_TOKEN }}@github.com/googleapis/googleapis-gen.git + API_SHORTNAME: ${{ github.event.inputs.api_shortname }} + NAME_PRETTY: ${{ github.event.inputs.name_pretty }} + PROTO_PATH: ${{ github.event.inputs.proto_path }} + PRODUCT_DOCS: ${{ github.event.inputs.product_docs }} + REST_DOCS: ${{ github.event.inputs.rest_docs }} + RPC_DOCS: ${{ github.event.inputs.rpc_docs }} + API_DESCRIPTION: ${{ github.event.inputs.api_description }} + RELEASE_LEVEL: ${{ github.event.inputs.release_level }} + TRANSPORT: ${{ github.event.inputs.transport }} + LANGUAGE: ${{ github.event.inputs.language }} + DISTRIBUTION_NAME: ${{ github.event.inputs.distribution_name }} + API_ID: ${{ github.event.inputs.api_id }} + REQUIRES_BILLING: ${{ github.event.inputs.requires_billing }} + DESTINATION_NAME: ${{ github.event.inputs.destination_name }} + CLOUD_API: ${{ github.event.inputs.cloud_api }} + GROUP_ID: ${{ github.event.inputs.group_id }} + OWLBOT_IMAGE: ${{ github.event.inputs.owlbot_image }} + LIBRARY_TYPE: ${{ github.event.inputs.library_type }} diff --git a/.github/workflows/verify-client-generation.yaml b/.github/workflows/verify-client-generation.yaml index 9656230ac854..5416de9f43f2 100644 --- a/.github/workflows/verify-client-generation.yaml +++ b/.github/workflows/verify-client-generation.yaml @@ -38,7 +38,7 @@ jobs: with: path: ~/.m2/repository key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }} - - name: Install dependencies + - name: Install new-client.py dependencies run: pip install --require-hashes -r generation/new_client/requirements.txt - name: Remove Client run: | From 2e1ef6a481783ba27807c0b3a794a2b80a30dbac Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 21:43:11 +0000 Subject: [PATCH 04/51] improve input description --- .github/workflows/generate_new_client.yaml | 38 +++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index eb4182432d3b..bd13bd4310c6 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -5,36 +5,36 @@ on: api_shortname: required: true type: string - description: Name for the new directory name and (default) artifact name + description: `api_shortname`: Name for the new directory name and (default) artifact name name_pretty: required: true type: string - description: The human-friendly name that appears in README.md + description: `name_pretty`: The human-friendly name that appears in README.md proto_path: required: true type: string description: | - Path to proto file from the root of the googleapis repository to the + `proto_path`: Path to proto file from the root of the googleapis repository to the directory that contains the proto files (without the version). For example, to generate the library for 'google/maps/routing/v2', then you specify this value as 'google/maps/routing' product_docs: required: true type: string - description: Documentation URL that appears in README.md + description: `product_docs`: Documentation URL that appears in README.md rest_docs: required: false type: string description: | - If it exists, link to the REST Documentation for a service + `rest_docs`: If it exists, link to the REST Documentation for a service rpc_docs: required: false type: string description: | - If it exists, link to the RPC Documentation for a service + `rpc_docs`: If it exists, link to the RPC Documentation for a service api_description: required: true - description: Description that appears in README.md + description: `api_description`: Description that appears in README.md release_level: required: false type: choice @@ -42,7 +42,7 @@ on: - stable - preview description: | - A label that appears in repo-metadata.json. + `release_level`: A label that appears in repo-metadata.json. The first library generation is always 'preview'. transport: required: false @@ -51,7 +51,7 @@ on: - grpc - http - both - description: A label that appears in repo-metadata.json + description: `transport`: A label that appears in repo-metadata.json language: required: false type: string @@ -60,46 +60,46 @@ on: required: false type: string description: | - Maven coordinates of the generated library. By default it's + `distribution_name`: Maven coordinates of the generated library. By default it's "com.google.cloud:google-cloud-" api_id: required: false type: string description: | - The value of the apiid parameter used in README.md It has link to + `api_id`: The value of the apiid parameter used in README.md It has link to "https://console.cloud.google.com/flows/enableapi?apiid=" requires_billling: required: false type: boolean description: | - Based on this value, README.md explains whether billing setup is - "needed or not." + `requires_billling`: Based on this value, README.md explains whether billing setup is + needed or not. destination_name: required: false type: string description: | - The directory name of the new library. By default it's - "java-" + `destination_name`: The directory name of the new library. By default it's + java- cloud_api: required: false type: boolean description: | - If true, the artifact ID of the library is 'google-cloud-' + `cloud_api`: If true, the artifact ID of the library is 'google-cloud-' otherwise 'google-' group_id: required: false type: string description: | - The group ID of the artifact when distribution name is not set + `group_id`: The group ID of the artifact when distribution name is not set owlbot_image: required: false type: string - description: The owlbot container image used in OwlBot.yaml + description: `owlbot_image`: The owlbot container image used in OwlBot.yaml library_type: required: false type: string description: | - A label that appear in repo-metadata.json to tell how the library is + `library_type`: A label that appear in repo-metadata.json to tell how the library is maintained or generated jobs: From f766b2c78a50d08dab1b0b312969f20882c237de Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 21:45:46 +0000 Subject: [PATCH 05/51] test correct backtick syntax --- .github/workflows/generate_new_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index bd13bd4310c6..147ae2c300ea 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -5,7 +5,7 @@ on: api_shortname: required: true type: string - description: `api_shortname`: Name for the new directory name and (default) artifact name + description: "`api_shortname`: Name for the new directory name and (default) artifact name" name_pretty: required: true type: string From 24a8b3958add33269aca5f539f01183bb47581bb Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 21:47:01 +0000 Subject: [PATCH 06/51] enclose backtick containing descriptions in double quotes --- .github/workflows/generate_new_client.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 147ae2c300ea..aafa77958532 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -9,7 +9,7 @@ on: name_pretty: required: true type: string - description: `name_pretty`: The human-friendly name that appears in README.md + description: "`name_pretty`: The human-friendly name that appears in README.md" proto_path: required: true type: string @@ -21,7 +21,7 @@ on: product_docs: required: true type: string - description: `product_docs`: Documentation URL that appears in README.md + description: "`product_docs`: Documentation URL that appears in README.md" rest_docs: required: false type: string @@ -34,7 +34,7 @@ on: `rpc_docs`: If it exists, link to the RPC Documentation for a service api_description: required: true - description: `api_description`: Description that appears in README.md + description: "`api_description`: Description that appears in README.md" release_level: required: false type: choice @@ -51,7 +51,7 @@ on: - grpc - http - both - description: `transport`: A label that appears in repo-metadata.json + description: "`transport`: A label that appears in repo-metadata.json" language: required: false type: string @@ -94,7 +94,7 @@ on: owlbot_image: required: false type: string - description: `owlbot_image`: The owlbot container image used in OwlBot.yaml + description: "`owlbot_image`: The owlbot container image used in OwlBot.yaml" library_type: required: false type: string From d1fe1191da0431fc7b8b8c1f915d0d118c1950f0 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 22:17:47 +0000 Subject: [PATCH 07/51] remove "less necessary" inputs --- .github/workflows/generate_new_client.yaml | 48 +--------------------- 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index aafa77958532..ef74ba1628e4 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -1,6 +1,7 @@ name: Generate new GAPIC client library on: workflow_dispatch: + # some inputs are ommited due to limit of 10 input arguments inputs: api_shortname: required: true @@ -35,15 +36,6 @@ on: api_description: required: true description: "`api_description`: Description that appears in README.md" - release_level: - required: false - type: choice - options: - - stable - - preview - description: | - `release_level`: A label that appears in repo-metadata.json. - The first library generation is always 'preview'. transport: required: false type: choice @@ -52,34 +44,6 @@ on: - http - both description: "`transport`: A label that appears in repo-metadata.json" - language: - required: false - type: string - default: java - distribution_name: - required: false - type: string - description: | - `distribution_name`: Maven coordinates of the generated library. By default it's - "com.google.cloud:google-cloud-" - api_id: - required: false - type: string - description: | - `api_id`: The value of the apiid parameter used in README.md It has link to - "https://console.cloud.google.com/flows/enableapi?apiid=" - requires_billling: - required: false - type: boolean - description: | - `requires_billling`: Based on this value, README.md explains whether billing setup is - needed or not. - destination_name: - required: false - type: string - description: | - `destination_name`: The directory name of the new library. By default it's - java- cloud_api: required: false type: boolean @@ -91,16 +55,6 @@ on: type: string description: | `group_id`: The group ID of the artifact when distribution name is not set - owlbot_image: - required: false - type: string - description: "`owlbot_image`: The owlbot container image used in OwlBot.yaml" - library_type: - required: false - type: string - description: | - `library_type`: A label that appear in repo-metadata.json to tell how the library is - maintained or generated jobs: generate: From 07d1762dae3815dae0a31353a020e59df43a2eb7 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 22:20:47 +0000 Subject: [PATCH 08/51] default values for non-string inputs --- .github/workflows/generate_new_client.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index ef74ba1628e4..92499e15bd0d 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -39,6 +39,7 @@ on: transport: required: false type: choice + default: grpc options: - grpc - http @@ -46,6 +47,7 @@ on: description: "`transport`: A label that appears in repo-metadata.json" cloud_api: required: false + default: true type: boolean description: | `cloud_api`: If true, the artifact ID of the library is 'google-cloud-' From a096c7c089ed813dd4d7765b2018f212170ef560 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 22:23:17 +0000 Subject: [PATCH 09/51] fix ubuntu version, remove java install --- .github/workflows/generate_new_client.yaml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 92499e15bd0d..ef5bac34d8ce 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -60,7 +60,7 @@ on: jobs: generate: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -70,21 +70,12 @@ jobs: - name: Get current week within the year id: date run: echo "::set-output name=week_of_year::$(date +'%W' --utc)" - - uses: actions/setup-java@v3 - with: - distribution: zulu - java-version: 11 - - run: java -version - - uses: actions/cache@v3 - id: mvn-cache - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }} - name: Install new-client.py dependencies run: pip install --require-hashes -r generation/new_client/requirements.txt - run: echo 'a message' - name: Generate run: | + set -x arguments="--api_shortname=${API_SHORTNAME} \ --proto_path=${PROTO_PATH} \ --name-pretty=${NAME_PRETTY} \ From b61577ed7d4696f74f5a617f906582bb675094ea Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 22:34:42 +0000 Subject: [PATCH 10/51] fix xargs and arguments --- .github/workflows/generate_new_client.yaml | 30 ++++++++-------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index ef5bac34d8ce..7f815faabbb7 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -76,12 +76,12 @@ jobs: - name: Generate run: | set -x - arguments="--api_shortname=${API_SHORTNAME} \ - --proto_path=${PROTO_PATH} \ - --name-pretty=${NAME_PRETTY} \ - --product-docs=${PRODUCT_DOCS} \ - --api_description=${API_DESCRIPTION} | - --googleapis-gen-url=${GOOGLEAPIS_GEN_URL}" + arguments="--api_shortname=\"${API_SHORTNAME}\" \ + --proto_path=\"${PROTO_PATH}\" \ + --name-pretty=\"${NAME_PRETTY}\" \ + --product-docs=\"${PRODUCT_DOCS}\" \ + --api_description=\"${API_DESCRIPTION}\" \ + --googleapis-gen-url=\"${GOOGLEAPIS_GEN_URL}\"" # helper function that appends a python argument only if specified in the GH action inputs append_argument() { @@ -90,18 +90,16 @@ jobs: env_name=$(echo "${py_arg}" | sed 's/-/_/g' | sed -e 's/\([a-z]\)/\U\1/g') if [[ -n "${!env_name}" ]]; then # $(echo) is redundant but it works around a syntax highlighting problem in vim - arguments=$(echo "${arguments} --${py_arg}=${!env_name}") + arguments=$(echo "${arguments} --${py_arg}=\"${!env_name}\"") fi } - declare -a optional_args=('release-level' 'transport' 'distribution-name' 'api-id' - 'requires-billing' 'destination-name' 'cloud-api' 'group-id' 'owlbot-image' - 'library-type' 'rest-docs' 'rpc-docs') + declare -a optional_args=('transport' 'cloud-api' 'group-id' 'rest-docs' 'rpc-docs') for python_argument in "${optional_args[@]}"; do append_argument "${python_argument}" done - echo "${arguments}" | xargs -0 python generation/new_client/new-client.py generate + echo "${arguments}" | xargs python generation/new_client/new-client.py generate env: GOOGLEAPIS_GEN_URL: https://yoshi-approver:${{ secrets.YOSHI_APPROVER_PRIVATE_TOKEN }}@github.com/googleapis/googleapis-gen.git API_SHORTNAME: ${{ github.event.inputs.api_shortname }} @@ -110,16 +108,8 @@ jobs: PRODUCT_DOCS: ${{ github.event.inputs.product_docs }} REST_DOCS: ${{ github.event.inputs.rest_docs }} RPC_DOCS: ${{ github.event.inputs.rpc_docs }} + TRANSPORT: ${{ github.event.inputs.transport }} API_DESCRIPTION: ${{ github.event.inputs.api_description }} - RELEASE_LEVEL: ${{ github.event.inputs.release_level }} TRANSPORT: ${{ github.event.inputs.transport }} - LANGUAGE: ${{ github.event.inputs.language }} - DISTRIBUTION_NAME: ${{ github.event.inputs.distribution_name }} - API_ID: ${{ github.event.inputs.api_id }} - REQUIRES_BILLING: ${{ github.event.inputs.requires_billing }} - DESTINATION_NAME: ${{ github.event.inputs.destination_name }} - CLOUD_API: ${{ github.event.inputs.cloud_api }} GROUP_ID: ${{ github.event.inputs.group_id }} - OWLBOT_IMAGE: ${{ github.event.inputs.owlbot_image }} - LIBRARY_TYPE: ${{ github.event.inputs.library_type }} From 6a53ab95caeecb14a6f54526a54912035f90512b Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 22:36:28 +0000 Subject: [PATCH 11/51] fix input-to-env list --- .github/workflows/generate_new_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 7f815faabbb7..804c2ea57aeb 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -108,8 +108,8 @@ jobs: PRODUCT_DOCS: ${{ github.event.inputs.product_docs }} REST_DOCS: ${{ github.event.inputs.rest_docs }} RPC_DOCS: ${{ github.event.inputs.rpc_docs }} - TRANSPORT: ${{ github.event.inputs.transport }} API_DESCRIPTION: ${{ github.event.inputs.api_description }} TRANSPORT: ${{ github.event.inputs.transport }} + CLOUD_API: ${{ github.event.inputs.CLOUD_API }} GROUP_ID: ${{ github.event.inputs.group_id }} From 10f906886813c7657b8fd56d963414e7db8c8bf6 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 22:40:30 +0000 Subject: [PATCH 12/51] correct underscores in input to new-client --- .github/workflows/generate_new_client.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 804c2ea57aeb..457d11df1200 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -77,10 +77,10 @@ jobs: run: | set -x arguments="--api_shortname=\"${API_SHORTNAME}\" \ - --proto_path=\"${PROTO_PATH}\" \ + --proto-path=\"${PROTO_PATH}\" \ --name-pretty=\"${NAME_PRETTY}\" \ --product-docs=\"${PRODUCT_DOCS}\" \ - --api_description=\"${API_DESCRIPTION}\" \ + --api-description=\"${API_DESCRIPTION}\" \ --googleapis-gen-url=\"${GOOGLEAPIS_GEN_URL}\"" # helper function that appends a python argument only if specified in the GH action inputs From 2da7fbf85ef22cdaee709414b00cee49f6538f7e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 23:00:43 +0000 Subject: [PATCH 13/51] add PR creation --- .github/workflows/generate_new_client.yaml | 32 ++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 457d11df1200..2bb04949d875 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -74,14 +74,15 @@ jobs: run: pip install --require-hashes -r generation/new_client/requirements.txt - run: echo 'a message' - name: Generate + if: generation run: | set -x arguments="--api_shortname=\"${API_SHORTNAME}\" \ - --proto-path=\"${PROTO_PATH}\" \ - --name-pretty=\"${NAME_PRETTY}\" \ - --product-docs=\"${PRODUCT_DOCS}\" \ - --api-description=\"${API_DESCRIPTION}\" \ - --googleapis-gen-url=\"${GOOGLEAPIS_GEN_URL}\"" + --proto-path=\"${PROTO_PATH}\" \ + --name-pretty=\"${NAME_PRETTY}\" \ + --product-docs=\"${PRODUCT_DOCS}\" \ + --api-description=\"${API_DESCRIPTION}\" \ + --googleapis-gen-url=\"${GOOGLEAPIS_GEN_URL}\"" # helper function that appends a python argument only if specified in the GH action inputs append_argument() { @@ -99,6 +100,7 @@ jobs: for python_argument in "${optional_args[@]}"; do append_argument "${python_argument}" done + echo "::set-output name=new_library_args::${arguments}" echo "${arguments}" | xargs python generation/new_client/new-client.py generate env: GOOGLEAPIS_GEN_URL: https://yoshi-approver:${{ secrets.YOSHI_APPROVER_PRIVATE_TOKEN }}@github.com/googleapis/googleapis-gen.git @@ -112,4 +114,24 @@ jobs: TRANSPORT: ${{ github.event.inputs.transport }} CLOUD_API: ${{ github.event.inputs.CLOUD_API }} GROUP_ID: ${{ github.event.inputs.group_id }} + - name: Push to branch + run: | + [ -z "`git config user.email`" ] && git config --global user.email "${USERNAME:-script}@google.com" + [ -z "`git config user.name`" ] && git config --global user.name "${USERNAME:-script}" + + # create and push to branch in origin + branch_name="new-library/${{ github.event.inputs.api_shortname }}" + git checkout -b "${branch_name}" + git add --all + git commit -m "feat: [${API_SHORTNAME}] new module for ${API_SHORT_NAME}\n\n\ + \`\`\`${GENERATION_ARGUMENTS}\`\`\`" + git remote add monorepo https://${{ github.actor }}:${{ github.token }}@github.com/${{ github.repository }}.git + git fetch -q --unshallow monorepo + git push -f monorepo "${branch_name}" + + # create PR + gh pr create --fill + env: + API_SHORTNAME: ${{ github.event.inputs.api_shortname }} + GENERATION_ARGUMENTS: ${{ steps.generation.outputs.new_library_args }} From 8855e9a7cdb3ee27ab78e8069e5db7f9126a89dd Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 23:01:44 +0000 Subject: [PATCH 14/51] fix id typo --- .github/workflows/generate_new_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 2bb04949d875..225091fd966c 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -74,7 +74,7 @@ jobs: run: pip install --require-hashes -r generation/new_client/requirements.txt - run: echo 'a message' - name: Generate - if: generation + id: generation run: | set -x arguments="--api_shortname=\"${API_SHORTNAME}\" \ From 9b65c2b2f578021285ad4a56727d26769253fab4 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 12 Jan 2024 23:23:27 +0000 Subject: [PATCH 15/51] add gh token to pr creation step --- .github/workflows/generate_new_client.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 225091fd966c..e67a5646e39a 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -134,4 +134,5 @@ jobs: env: API_SHORTNAME: ${{ github.event.inputs.api_shortname }} GENERATION_ARGUMENTS: ${{ steps.generation.outputs.new_library_args }} + GH_TOKEN: ${{ secrets.YOSHI_APPROVER_PRIVATE_TOKEN }} From 9e286e9089c086056783f11c9c6ad4d5bb95e0d8 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Sun, 14 Jan 2024 22:37:10 +0000 Subject: [PATCH 16/51] use YOSHI_CODE_BOT_TOKEN --- .github/workflows/generate_new_client.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index e67a5646e39a..df10a23dfb39 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -103,7 +103,7 @@ jobs: echo "::set-output name=new_library_args::${arguments}" echo "${arguments}" | xargs python generation/new_client/new-client.py generate env: - GOOGLEAPIS_GEN_URL: https://yoshi-approver:${{ secrets.YOSHI_APPROVER_PRIVATE_TOKEN }}@github.com/googleapis/googleapis-gen.git + GOOGLEAPIS_GEN_URL: https://yoshi-approver:${{ secrets.YOSHI_CODE_BOT_TOKEN }}@github.com/googleapis/googleapis-gen.git API_SHORTNAME: ${{ github.event.inputs.api_shortname }} NAME_PRETTY: ${{ github.event.inputs.name_pretty }} PROTO_PATH: ${{ github.event.inputs.proto_path }} @@ -134,5 +134,5 @@ jobs: env: API_SHORTNAME: ${{ github.event.inputs.api_shortname }} GENERATION_ARGUMENTS: ${{ steps.generation.outputs.new_library_args }} - GH_TOKEN: ${{ secrets.YOSHI_APPROVER_PRIVATE_TOKEN }} + GH_TOKEN: ${{ secrets.YOSHI_CODE_BOT_TOKEN }} From b93107e402d89e5cbd20821c5e213aa33d0a7aa5 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Sun, 14 Jan 2024 22:40:54 +0000 Subject: [PATCH 17/51] use distribution_name instead of group_id --- .github/workflows/generate_new_client.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index df10a23dfb39..460804c3b90f 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -52,12 +52,12 @@ on: description: | `cloud_api`: If true, the artifact ID of the library is 'google-cloud-' otherwise 'google-' - group_id: + distribution_name: required: false type: string description: | - `group_id`: The group ID of the artifact when distribution name is not set - + `distribution_name`: Maven coordinates of the generated library. By default it's + com.google.cloud:google-cloud- jobs: generate: runs-on: ubuntu-22.04 @@ -114,7 +114,7 @@ jobs: TRANSPORT: ${{ github.event.inputs.transport }} CLOUD_API: ${{ github.event.inputs.CLOUD_API }} GROUP_ID: ${{ github.event.inputs.group_id }} - - name: Push to branch + - name: Push to branch and create PR run: | [ -z "`git config user.email`" ] && git config --global user.email "${USERNAME:-script}@google.com" [ -z "`git config user.name`" ] && git config --global user.name "${USERNAME:-script}" From c869604520a7729b8d2670fd3df21447d0b38772 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Sun, 14 Jan 2024 22:55:29 +0000 Subject: [PATCH 18/51] use debug output --- .github/workflows/generate_new_client.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 460804c3b90f..ad3966d44d57 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -116,6 +116,7 @@ jobs: GROUP_ID: ${{ github.event.inputs.group_id }} - name: Push to branch and create PR run: | + set -x [ -z "`git config user.email`" ] && git config --global user.email "${USERNAME:-script}@google.com" [ -z "`git config user.name`" ] && git config --global user.name "${USERNAME:-script}" @@ -127,7 +128,7 @@ jobs: \`\`\`${GENERATION_ARGUMENTS}\`\`\`" git remote add monorepo https://${{ github.actor }}:${{ github.token }}@github.com/${{ github.repository }}.git git fetch -q --unshallow monorepo - git push -f monorepo "${branch_name}" + git push --head -f monorepo "${branch_name}" # create PR gh pr create --fill From 7ea2f6443ae005cda151d51f48fed3dbac3ed29d Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Sun, 14 Jan 2024 23:17:41 +0000 Subject: [PATCH 19/51] remove wrong push option --- .github/workflows/generate_new_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index ad3966d44d57..a8c940fd9b14 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -128,7 +128,7 @@ jobs: \`\`\`${GENERATION_ARGUMENTS}\`\`\`" git remote add monorepo https://${{ github.actor }}:${{ github.token }}@github.com/${{ github.repository }}.git git fetch -q --unshallow monorepo - git push --head -f monorepo "${branch_name}" + git push -f monorepo "${branch_name}" # create PR gh pr create --fill From 82229256637d4c4a59520bd8238543928942e1af Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 00:02:53 +0000 Subject: [PATCH 20/51] sanitize token commit message --- .github/workflows/generate_new_client.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index a8c940fd9b14..ef2d8b9de3ea 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -81,8 +81,7 @@ jobs: --proto-path=\"${PROTO_PATH}\" \ --name-pretty=\"${NAME_PRETTY}\" \ --product-docs=\"${PRODUCT_DOCS}\" \ - --api-description=\"${API_DESCRIPTION}\" \ - --googleapis-gen-url=\"${GOOGLEAPIS_GEN_URL}\"" + --api-description=\"${API_DESCRIPTION}\"" # helper function that appends a python argument only if specified in the GH action inputs append_argument() { @@ -101,7 +100,7 @@ jobs: append_argument "${python_argument}" done echo "::set-output name=new_library_args::${arguments}" - echo "${arguments}" | xargs python generation/new_client/new-client.py generate + echo "${arguments} --googleapis-gen-url=\"${GOOGLEAPIS_GEN_URL}\"" | xargs python generation/new_client/new-client.py generate env: GOOGLEAPIS_GEN_URL: https://yoshi-approver:${{ secrets.YOSHI_CODE_BOT_TOKEN }}@github.com/googleapis/googleapis-gen.git API_SHORTNAME: ${{ github.event.inputs.api_shortname }} From 7e9162d65eda853e53f7635a31c01d45fb37ad9c Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 00:19:48 +0000 Subject: [PATCH 21/51] specify branch in gh pr create --- .github/workflows/generate_new_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index ef2d8b9de3ea..c90345c5f394 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -130,7 +130,7 @@ jobs: git push -f monorepo "${branch_name}" # create PR - gh pr create --fill + gh pr create --fill --head "${branch_name}" env: API_SHORTNAME: ${{ github.event.inputs.api_shortname }} GENERATION_ARGUMENTS: ${{ steps.generation.outputs.new_library_args }} From 03e746d6957e5797f5364af5b2dee5f067d9ed39 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 00:41:09 +0000 Subject: [PATCH 22/51] escape newlines for pr message --- .github/workflows/generate_new_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index c90345c5f394..d4db795fe5fb 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -123,7 +123,7 @@ jobs: branch_name="new-library/${{ github.event.inputs.api_shortname }}" git checkout -b "${branch_name}" git add --all - git commit -m "feat: [${API_SHORTNAME}] new module for ${API_SHORT_NAME}\n\n\ + git commit -m "feat: [${API_SHORTNAME}] new module for ${API_SHORT_NAME}\\n\\n\ \`\`\`${GENERATION_ARGUMENTS}\`\`\`" git remote add monorepo https://${{ github.actor }}:${{ github.token }}@github.com/${{ github.repository }}.git git fetch -q --unshallow monorepo From 72cfefaa01f2d0d91895ba0062df1c5e8d0a06f4 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 04:36:22 +0000 Subject: [PATCH 23/51] try different newline approach --- .github/workflows/generate_new_client.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index d4db795fe5fb..750a0099f411 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -123,7 +123,9 @@ jobs: branch_name="new-library/${{ github.event.inputs.api_shortname }}" git checkout -b "${branch_name}" git add --all - git commit -m "feat: [${API_SHORTNAME}] new module for ${API_SHORT_NAME}\\n\\n\ + git commit -m "feat: [${API_SHORTNAME}] new module for ${API_SHORTNAME} + + \`\`\`${GENERATION_ARGUMENTS}\`\`\`" git remote add monorepo https://${{ github.actor }}:${{ github.token }}@github.com/${{ github.repository }}.git git fetch -q --unshallow monorepo From 4299d4b111fddee763fe423a1c1da19a4a090533 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 18:50:58 +0000 Subject: [PATCH 24/51] improve pr description --- .github/workflows/generate_new_client.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 750a0099f411..4f1037466c66 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -126,7 +126,9 @@ jobs: git commit -m "feat: [${API_SHORTNAME}] new module for ${API_SHORTNAME} - \`\`\`${GENERATION_ARGUMENTS}\`\`\`" + \`\`\` + python generation/new_client/new-client.py generate ${GENERATION_ARGUMENTS} + \`\`\`" git remote add monorepo https://${{ github.actor }}:${{ github.token }}@github.com/${{ github.repository }}.git git fetch -q --unshallow monorepo git push -f monorepo "${branch_name}" From 6611ec191a38d5b799cbaaeea70820d5be077fac Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 18:58:24 +0000 Subject: [PATCH 25/51] correct username to author the commits --- .github/workflows/generate_new_client.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 4f1037466c66..9ec4da8fd236 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -136,6 +136,7 @@ jobs: # create PR gh pr create --fill --head "${branch_name}" env: + USERNAME: ${{ github.actor }} API_SHORTNAME: ${{ github.event.inputs.api_shortname }} GENERATION_ARGUMENTS: ${{ steps.generation.outputs.new_library_args }} GH_TOKEN: ${{ secrets.YOSHI_CODE_BOT_TOKEN }} From 4d2cebb52c262f74761d993cdf8712e9987e660e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 19:16:15 +0000 Subject: [PATCH 26/51] use destination_name argument instead of cloud_api --- .github/workflows/generate_new_client.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 9ec4da8fd236..40080a3dbd16 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -45,13 +45,12 @@ on: - http - both description: "`transport`: A label that appears in repo-metadata.json" - cloud_api: + destination_name: required: false - default: true - type: boolean + type: string description: | - `cloud_api`: If true, the artifact ID of the library is 'google-cloud-' - otherwise 'google-' + `destination_name`: The directory name of the new library. By default it's + java- distribution_name: required: false type: string From 4c57f8fab041e913f36b90fa579e54bdb44b84f7 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 19:17:51 +0000 Subject: [PATCH 27/51] prepare `destination-name` argument --- .github/workflows/generate_new_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 40080a3dbd16..7d172f0c3987 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -93,7 +93,7 @@ jobs: fi } - declare -a optional_args=('transport' 'cloud-api' 'group-id' 'rest-docs' 'rpc-docs') + declare -a optional_args=('transport' 'destination-name' 'group-id' 'rest-docs' 'rpc-docs') for python_argument in "${optional_args[@]}"; do append_argument "${python_argument}" From 538b16d80d08e4629e236d3ba9d1dba3e430f696 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 19:34:09 +0000 Subject: [PATCH 28/51] correct env var setup --- .github/workflows/generate_new_client.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 7d172f0c3987..2615997eb6e3 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -110,8 +110,8 @@ jobs: RPC_DOCS: ${{ github.event.inputs.rpc_docs }} API_DESCRIPTION: ${{ github.event.inputs.api_description }} TRANSPORT: ${{ github.event.inputs.transport }} - CLOUD_API: ${{ github.event.inputs.CLOUD_API }} - GROUP_ID: ${{ github.event.inputs.group_id }} + DESTINATION_NAME: ${{ github.event.inputs.destination_name }} + DISTRIBUTION_NAME: ${{ github.event.inputs.distribution_name }} - name: Push to branch and create PR run: | set -x From 4bb1ddafae55784c9285d30f4e49b526f0f95871 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 19:37:09 +0000 Subject: [PATCH 29/51] randomize branch name --- .github/workflows/generate_new_client.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 2615997eb6e3..10a71b15db61 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -119,7 +119,9 @@ jobs: [ -z "`git config user.name`" ] && git config --global user.name "${USERNAME:-script}" # create and push to branch in origin - branch_name="new-library/${{ github.event.inputs.api_shortname }}" + # random_id allows multiple runs of this workflow + random_id=(tr -dc A-Za-z0-9 Date: Mon, 15 Jan 2024 20:23:59 +0000 Subject: [PATCH 30/51] correct random_id syntax --- .github/workflows/generate_new_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 10a71b15db61..16e5960cc7de 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -120,7 +120,7 @@ jobs: # create and push to branch in origin # random_id allows multiple runs of this workflow - random_id=(tr -dc A-Za-z0-9 Date: Mon, 15 Jan 2024 22:19:15 +0000 Subject: [PATCH 31/51] add distribution-name to arguments --- .github/workflows/generate_new_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 16e5960cc7de..6ed8bf5df0d1 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -93,7 +93,7 @@ jobs: fi } - declare -a optional_args=('transport' 'destination-name' 'group-id' 'rest-docs' 'rpc-docs') + declare -a optional_args=('transport' 'destination-name' 'distribution-name' 'group-id' 'rest-docs' 'rpc-docs') for python_argument in "${optional_args[@]}"; do append_argument "${python_argument}" From 9319d8b200f1f9195b9f5a0fc1b277cc59fa89b1 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 15 Jan 2024 23:22:11 +0000 Subject: [PATCH 32/51] remove dummy message step --- .github/workflows/generate_new_client.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 6ed8bf5df0d1..1bc2fddeacd5 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -71,7 +71,6 @@ jobs: run: echo "::set-output name=week_of_year::$(date +'%W' --utc)" - name: Install new-client.py dependencies run: pip install --require-hashes -r generation/new_client/requirements.txt - - run: echo 'a message' - name: Generate id: generation run: | From 63607c436076acf4d68b7004585796cbbab6f14b Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 15:32:32 +0000 Subject: [PATCH 33/51] include workflow location in commit message --- .github/workflows/generate_new_client.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index 1bc2fddeacd5..c73a11bfaa94 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -126,6 +126,10 @@ jobs: git commit -m "feat: [${API_SHORTNAME}] new module for ${API_SHORTNAME} + Generated with `.github/workflows/generate_new_client.yaml` + + Command used: + \`\`\` python generation/new_client/new-client.py generate ${GENERATION_ARGUMENTS} \`\`\`" From 1d22d73c6a166dcabd1ddcaefaeee22ea1d5b5f3 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 15:39:28 +0000 Subject: [PATCH 34/51] use url for workflow link --- .github/workflows/generate_new_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client.yaml b/.github/workflows/generate_new_client.yaml index c73a11bfaa94..422a3b6c8993 100644 --- a/.github/workflows/generate_new_client.yaml +++ b/.github/workflows/generate_new_client.yaml @@ -126,7 +126,7 @@ jobs: git commit -m "feat: [${API_SHORTNAME}] new module for ${API_SHORTNAME} - Generated with `.github/workflows/generate_new_client.yaml` + Generated with https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml Command used: From 5830b219d72c8288f98cc8bd0c3ff376616cacb3 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 16:34:27 +0000 Subject: [PATCH 35/51] update new-client README --- generation/new_client/README.md | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 41e7d5b638ca..dd39d0a97da0 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -5,12 +5,37 @@ google-cloud-java monorepo. **This tool is for repository maintainers only. Not for library users.** -## Prerequisites - -This section is only needed for the first run of this script. If it's already +## Run via github action + +You can save the time to setup the environment by calling the +[`generate_new_client.yaml` github action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) directly. You can jump to +[this section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#run-client-generation-script) +to find workflow arguments. + +> ![IMPORTANT] +> Not all the `new-client.py` arguments are available in the Github Action. +> Please refer to +> [this +> section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) +> for more arguments (it requires to setup a local environment). +> The arguments currently supported by the workflow are: +> - [API short Name (`api_shortname`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#api-short-name) +> - [Proto path (`proto_path`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#proto-path) +> - [Name pretty (`name_pretty`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#name-pretty) +> - [Product Docs (`product_docs`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#product-docs) +> - [REST Docs (`rest_docs`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#rest-docs) +> - [RPC Docs (`rpc_docs`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#rpc-docs) +> - [API description (`api_description`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#api-description) +> - [`transport`](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) +> - [`destination_name`](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) +> - [`distribution_name`](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) + + +## Prerequisites (for local environment) + +This section is only needed for the first _local_ run of this script. If it's already done, go to "Run client generation script" section. - ### Environment Use Linux environment. From 63bf9e635383bff64914ce46cd492d1708cfcdbf Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 20:15:05 +0000 Subject: [PATCH 36/51] move local prerequisites to bottom --- generation/new_client/README.md | 216 ++++++++++++++++---------------- 1 file changed, 109 insertions(+), 107 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index dd39d0a97da0..d3ef84aa2a5f 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -12,7 +12,7 @@ You can save the time to setup the environment by calling the [this section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#run-client-generation-script) to find workflow arguments. -> ![IMPORTANT] +![IMPORTANT] > Not all the `new-client.py` arguments are available in the Github Action. > Please refer to > [this @@ -30,17 +30,7 @@ to find workflow arguments. > - [`destination_name`](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) > - [`distribution_name`](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) - -## Prerequisites (for local environment) - -This section is only needed for the first _local_ run of this script. If it's already -done, go to "Run client generation script" section. - -### Environment - -Use Linux environment. - -Install Docker. +## Prerequisites (for GH action and local environment) ### Ensure no Release Please "snapshot" pull request open @@ -55,107 +45,13 @@ Background: This new client library generation process creates pom.xml files wit a "-SNAPSHOT" version. To have consistency with other modules, ensure the pom.xml files in the repository has "-SNAPSHOT" versions too. - -### Checkout google-cloud-java repository - -``` -$ git clone https://github.com/googleapis/google-cloud-java -$ git checkout main -$ git pull -``` - -### Install pyenv - -Install pyenv - -``` -curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer \ -| bash -``` - -Append the following lines to `$HOME/.bashrc`. - -``` -export PYENV_ROOT="$HOME/.pyenv" -command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" -eval "$(pyenv init -)" -eval "$(pyenv virtualenv-init -)" -``` - -Logout the shell and login again. You should be at the home directory. - -Assuming you have the following folder structure: -``` -~ (Home) - -> IdeaProjects/ - -> google-cloud-java - -> ... -``` -You can run these next commands in the home directory (or IdeaProjects). Otherwise, run it at `google-cloud-java`'s parent directory. - -Confirm pyenv installation succeeded: - -``` -~$ pyenv -pyenv 2.3.4 -Usage: pyenv [] - -Some useful pyenv commands are: - activate Activate virtual environment - commands List all available pyenv commands - deactivate Deactivate virtual environment -... -``` - -### Install Python 3.9 via pyenv - -``` -~$ pyenv install 3.9.13 -Downloading Python-3.9.13.tar.xz... --> https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tar.xz -Installing Python-3.9.13... -WARNING: The Python sqlite3 extension was not compiled. Missing the SQLite3 lib? -Installed Python-3.9.13 to /usr/local/google/home/suztomo/.pyenv/versions/3.9.13 -``` - -### Install Python v3.9.13 locally - -Run this command - -``` -$ pyenv local 3.9.13 -``` - -Confirm `python3.9` is installed: -``` -$ which python3.9 -/usr/local/google/home/suztomo/.pyenv/shims/python3.9 -``` - -### Install Python packages - -At the root of google-cloud-java repository clone, run: - -``` -$ python3.9 -m pip install -r generation/new_client/requirements.txt -``` - -### Install GitHub CLI (Optional) - -Install the GitHub CLI and login, if needed (may help with `Common Errors` below): - -``` -$ sudo apt-get install gh -$ gh auth login -``` - ## Double check that the library does not already exist within `google-cloud-java`! Some languages require a new request when a new version of a service is created, but Java manages all versions of the service as a single package, so the client library will automatically be updated to include new versions. ## Run client generation script -You will run new-client.py script with the following parameters. +You will run new-client.py script or github action with the following parameters. These parameters will be available in the Cloud Drop link (a YAML file) included in the buganizer request. The example in this README uses AlloyDB's [Cloud Drop](https://github.com/googleapis/googleapis/blob/master/google/cloud/alloydb/v1/alloydb_v1.yaml) file as an example. @@ -384,6 +280,112 @@ The script should finish creating a pull request even when the newly created module fails to compile. This gives the user flexibility to fix things in the created pull request. +## Prerequisites (for local environment) + +This section is only needed for the first _local_ run of this script. If it's already +done, go to "Run client generation script" section. + +### Environment + +Use Linux environment. + +Install Docker. + + + +### Checkout google-cloud-java repository + +``` +$ git clone https://github.com/googleapis/google-cloud-java +$ git checkout main +$ git pull +``` + +### Install pyenv + +Install pyenv + +``` +curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer \ +| bash +``` + +Append the following lines to `$HOME/.bashrc`. + +``` +export PYENV_ROOT="$HOME/.pyenv" +command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +``` + +Logout the shell and login again. You should be at the home directory. + +Assuming you have the following folder structure: +``` +~ (Home) + -> IdeaProjects/ + -> google-cloud-java + -> ... +``` +You can run these next commands in the home directory (or IdeaProjects). Otherwise, run it at `google-cloud-java`'s parent directory. + +Confirm pyenv installation succeeded: + +``` +~$ pyenv +pyenv 2.3.4 +Usage: pyenv [] + +Some useful pyenv commands are: + activate Activate virtual environment + commands List all available pyenv commands + deactivate Deactivate virtual environment +... +``` + +### Install Python 3.9 via pyenv + +``` +~$ pyenv install 3.9.13 +Downloading Python-3.9.13.tar.xz... +-> https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tar.xz +Installing Python-3.9.13... +WARNING: The Python sqlite3 extension was not compiled. Missing the SQLite3 lib? +Installed Python-3.9.13 to /usr/local/google/home/suztomo/.pyenv/versions/3.9.13 +``` + +### Install Python v3.9.13 locally + +Run this command + +``` +$ pyenv local 3.9.13 +``` + +Confirm `python3.9` is installed: +``` +$ which python3.9 +/usr/local/google/home/suztomo/.pyenv/shims/python3.9 +``` + +### Install Python packages + +At the root of google-cloud-java repository clone, run: + +``` +$ python3.9 -m pip install -r generation/new_client/requirements.txt +``` + +### Install GitHub CLI (Optional) + +Install the GitHub CLI and login, if needed (may help with `Common Errors` below): + +``` +$ sudo apt-get install gh +$ gh auth login +``` + # Common Errors ## Unable to clone googleapis-gen From f4e8d7130c0bb2a1e1a3c3904d796138e280ca01 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 20:18:21 +0000 Subject: [PATCH 37/51] explain advanced options not called in gh action --- generation/new_client/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index d3ef84aa2a5f..3c7e524f6b8b 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -170,6 +170,11 @@ command you ran above. For the explanation of the available parameters, run: `python3.9 generation/new_client/new-client.py generate --help`. +**Note: The advanced options other than `transport`, `destination-name` and +`distribution-name` cannot be specified in the GH action, you will have to run +the script locally (refer to the Prerequisites (for local environment) section +below)** + ``` ~/google-cloud-java$ python3.9 generation/new_client/new-client.py generate --help /usr/local/google/home/suztomo/google-cloud-java/generation/new_client From 741e7bd7d5c8bcfb9e0387f6318e300dda80ca6d Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 20:19:37 +0000 Subject: [PATCH 38/51] clarify advanced options note --- generation/new_client/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 3c7e524f6b8b..347bfac61174 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -173,7 +173,7 @@ For the explanation of the available parameters, run: **Note: The advanced options other than `transport`, `destination-name` and `distribution-name` cannot be specified in the GH action, you will have to run the script locally (refer to the Prerequisites (for local environment) section -below)** +below) if you need to specify any of these** ``` ~/google-cloud-java$ python3.9 generation/new_client/new-client.py generate --help From ce0a74424eb61357ee76b3f5cf380d42b9a5e6bd Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 20:22:31 +0000 Subject: [PATCH 39/51] simplify readme local links --- generation/new_client/README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 347bfac61174..378e1356100d 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -12,23 +12,23 @@ You can save the time to setup the environment by calling the [this section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#run-client-generation-script) to find workflow arguments. -![IMPORTANT] +> ![IMPORTANT] > Not all the `new-client.py` arguments are available in the Github Action. > Please refer to > [this > section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) > for more arguments (it requires to setup a local environment). > The arguments currently supported by the workflow are: -> - [API short Name (`api_shortname`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#api-short-name) -> - [Proto path (`proto_path`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#proto-path) -> - [Name pretty (`name_pretty`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#name-pretty) -> - [Product Docs (`product_docs`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#product-docs) -> - [REST Docs (`rest_docs`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#rest-docs) -> - [RPC Docs (`rpc_docs`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#rpc-docs) -> - [API description (`api_description`)](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#api-description) -> - [`transport`](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) -> - [`destination_name`](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) -> - [`distribution_name`](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) +> - [API short Name (`api_shortname`)](#api-short-name) +> - [Proto path (`proto_path`)](#proto-path) +> - [Name pretty (`name_pretty`)](#name-pretty) +> - [Product Docs (`product_docs`)](#product-docs) +> - [REST Docs (`rest_docs`)](#rest-docs) +> - [RPC Docs (`rpc_docs`)](#rpc-docs) +> - [API description (`api_description`)](#api-description) +> - [`transport`](#advanced-options) +> - [`destination_name`](#advanced-options) +> - [`distribution_name`](#advanced-options) ## Prerequisites (for GH action and local environment) From c3ca1d260811bb9998697323ec7e108ab45db59b Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 20:24:20 +0000 Subject: [PATCH 40/51] use alternative warning format --- generation/new_client/README.md | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 378e1356100d..82bf1a1809b7 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -12,23 +12,23 @@ You can save the time to setup the environment by calling the [this section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#run-client-generation-script) to find workflow arguments. -> ![IMPORTANT] -> Not all the `new-client.py` arguments are available in the Github Action. -> Please refer to -> [this -> section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) -> for more arguments (it requires to setup a local environment). -> The arguments currently supported by the workflow are: -> - [API short Name (`api_shortname`)](#api-short-name) -> - [Proto path (`proto_path`)](#proto-path) -> - [Name pretty (`name_pretty`)](#name-pretty) -> - [Product Docs (`product_docs`)](#product-docs) -> - [REST Docs (`rest_docs`)](#rest-docs) -> - [RPC Docs (`rpc_docs`)](#rpc-docs) -> - [API description (`api_description`)](#api-description) -> - [`transport`](#advanced-options) -> - [`destination_name`](#advanced-options) -> - [`distribution_name`](#advanced-options) +:warning: **IMPORTANT:** +Not all the `new-client.py` arguments are available in the Github Action. +Please refer to +[this +section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) +for more arguments (it requires to setup a local environment). +The arguments currently supported by the workflow are: +- [API short Name (`api_shortname`)](#api-short-name) +- [Proto path (`proto_path`)](#proto-path) +- [Name pretty (`name_pretty`)](#name-pretty) +- [Product Docs (`product_docs`)](#product-docs) +- [REST Docs (`rest_docs`)](#rest-docs) +- [RPC Docs (`rpc_docs`)](#rpc-docs) +- [API description (`api_description`)](#api-description) +- [`transport`](#advanced-options) +- [`destination_name`](#advanced-options) +- [`distribution_name`](#advanced-options) ## Prerequisites (for GH action and local environment) From a9dcd708bc604814cc485a7a1de921ff4cdcd7a6 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 20:26:17 +0000 Subject: [PATCH 41/51] move local example to the bottom --- generation/new_client/README.md | 63 +++++++++++++++++---------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 82bf1a1809b7..82a6cd31aac5 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -133,37 +133,6 @@ Example: ``` -### Example arguments - -Run `new-client.py` with the arguments above: - -``` -$ python3.9 generation/new_client/new-client.py generate \ - --api_shortname=alloydb \ - --proto-path=google/cloud/alloydb \ - --name-pretty="AlloyDB API" \ - --product-docs="https://cloud.google.com/alloydb/docs" \ - --api-description="AlloyDB for PostgreSQL is an open source-compatible database service that - provides a powerful option for migrating, modernizing, or building - commercial-grade applications." -``` - -The command creates changes for -the new module in the monorepo. At the end (~ 10 minutes), it tells you to -create a pull request in the monorepo: - -``` -... -Please create a pull request: - $ git checkout -b new_module_java-alloydb - $ git add . - $ git commit -m 'feat: [alloydb] new module for alloydb' - $ gh pr create --title 'feat: [alloydb] new module for alloydb' -``` - -Create a pull request from the change. -In the description, record the `python3.9 generation/new_client/new-client.py generate ...` -command you ran above. ## Advanced Options @@ -391,6 +360,38 @@ $ sudo apt-get install gh $ gh auth login ``` +## Example arguments for local call + +Run `new-client.py` with the arguments above: + +``` +$ python3.9 generation/new_client/new-client.py generate \ + --api_shortname=alloydb \ + --proto-path=google/cloud/alloydb \ + --name-pretty="AlloyDB API" \ + --product-docs="https://cloud.google.com/alloydb/docs" \ + --api-description="AlloyDB for PostgreSQL is an open source-compatible database service that + provides a powerful option for migrating, modernizing, or building + commercial-grade applications." +``` + +The command creates changes for +the new module in the monorepo. At the end (~ 10 minutes), it tells you to +create a pull request in the monorepo: + +``` +... +Please create a pull request: + $ git checkout -b new_module_java-alloydb + $ git add . + $ git commit -m 'feat: [alloydb] new module for alloydb' + $ gh pr create --title 'feat: [alloydb] new module for alloydb' +``` + +Create a pull request from the change. +In the description, record the `python3.9 generation/new_client/new-client.py generate ...` +command you ran above. + # Common Errors ## Unable to clone googleapis-gen From ac16f1d97445d00fd1ab4907079d8c8217557753 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 20:28:14 +0000 Subject: [PATCH 42/51] add links to gh action --- generation/new_client/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 82a6cd31aac5..ec61c510c0c1 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -51,7 +51,7 @@ Some languages require a new request when a new version of a service is created, ## Run client generation script -You will run new-client.py script or github action with the following parameters. +You will run new-client.py script or [github action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) with the following parameters. These parameters will be available in the Cloud Drop link (a YAML file) included in the buganizer request. The example in this README uses AlloyDB's [Cloud Drop](https://github.com/googleapis/googleapis/blob/master/google/cloud/alloydb/v1/alloydb_v1.yaml) file as an example. @@ -140,7 +140,7 @@ For the explanation of the available parameters, run: `python3.9 generation/new_client/new-client.py generate --help`. **Note: The advanced options other than `transport`, `destination-name` and -`distribution-name` cannot be specified in the GH action, you will have to run +`distribution-name` cannot be specified in the [GH action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml), you will have to run the script locally (refer to the Prerequisites (for local environment) section below) if you need to specify any of these** From 5d0e9c3300db91164525c51f7ad9472a2e8a95d8 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 20:29:35 +0000 Subject: [PATCH 43/51] highlight advanced options note with warning --- generation/new_client/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index ec61c510c0c1..f0226848c3c2 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -139,7 +139,7 @@ Example: For the explanation of the available parameters, run: `python3.9 generation/new_client/new-client.py generate --help`. -**Note: The advanced options other than `transport`, `destination-name` and +:warning: **Note: The advanced options other than `transport`, `destination-name` and `distribution-name` cannot be specified in the [GH action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml), you will have to run the script locally (refer to the Prerequisites (for local environment) section below) if you need to specify any of these** From b975972f9f120047de90a834304a15c1c178fb78 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 20:34:49 +0000 Subject: [PATCH 44/51] add explanation on finding the generated PR --- generation/new_client/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index f0226848c3c2..64a6152cff8f 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -30,6 +30,14 @@ The arguments currently supported by the workflow are: - [`destination_name`](#advanced-options) - [`distribution_name`](#advanced-options) +### Accessing the generated PR + +The workflow automatically creates a PR for you. +Once the workflow has run successfully, you can check the action logs +and see the **last line** of the _"Push to branch and create PR"_ step to +find a link to the generated pull +request. + ## Prerequisites (for GH action and local environment) ### Ensure no Release Please "snapshot" pull request open From 2fb82468e6f2269b01dd4ebcae36543de5a72f5d Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 16 Jan 2024 21:04:16 +0000 Subject: [PATCH 45/51] correct terminology/capitalization --- generation/new_client/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 64a6152cff8f..9850bdcef50c 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -5,10 +5,10 @@ google-cloud-java monorepo. **This tool is for repository maintainers only. Not for library users.** -## Run via github action +## Run via Github Action You can save the time to setup the environment by calling the -[`generate_new_client.yaml` github action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) directly. You can jump to +[`generate_new_client.yaml` Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) directly. You can jump to [this section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#run-client-generation-script) to find workflow arguments. @@ -38,9 +38,9 @@ and see the **last line** of the _"Push to branch and create PR"_ step to find a link to the generated pull request. -## Prerequisites (for GH action and local environment) +## Prerequisites (both for Github Action and local environment) -### Ensure no Release Please "snapshot" pull request open +### Ensure no Release Please "SNAPSHOT" pull request open Ensure google-cloud-java repository does not have [a pull request with "autorelease: snapshot" label]( https://github.com/googleapis/google-cloud-java/pulls?q=is%3Apr+is%3Aopen+label%3A%22autorelease%3A+snapshot%22). @@ -57,9 +57,9 @@ the repository has "-SNAPSHOT" versions too. Some languages require a new request when a new version of a service is created, but Java manages all versions of the service as a single package, so the client library will automatically be updated to include new versions. -## Run client generation script +## Execute the workflow (either locally or via Github Action) -You will run new-client.py script or [github action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) with the following parameters. +You will run new-client.py script or the [Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) with the following parameters. These parameters will be available in the Cloud Drop link (a YAML file) included in the buganizer request. The example in this README uses AlloyDB's [Cloud Drop](https://github.com/googleapis/googleapis/blob/master/google/cloud/alloydb/v1/alloydb_v1.yaml) file as an example. @@ -148,7 +148,7 @@ For the explanation of the available parameters, run: `python3.9 generation/new_client/new-client.py generate --help`. :warning: **Note: The advanced options other than `transport`, `destination-name` and -`distribution-name` cannot be specified in the [GH action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml), you will have to run +`distribution-name` cannot be specified in the [Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml), you will have to run the script locally (refer to the Prerequisites (for local environment) section below) if you need to specify any of these** @@ -265,7 +265,7 @@ created pull request. ## Prerequisites (for local environment) This section is only needed for the first _local_ run of this script. If it's already -done, go to "Run client generation script" section. +done, go to the "Execute the workflow (either locally or via Github Action)" section. ### Environment From 097a77853db51206b22d4304ef2fa5e7cc4aadc7 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 17 Jan 2024 16:01:53 +0000 Subject: [PATCH 46/51] improve top-to-bottom readability --- generation/new_client/README.md | 117 +++++++++++--------------------- 1 file changed, 41 insertions(+), 76 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 9850bdcef50c..549dc5421908 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -7,10 +7,10 @@ google-cloud-java monorepo. ## Run via Github Action -You can save the time to setup the environment by calling the -[`generate_new_client.yaml` Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) directly. You can jump to -[this section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#run-client-generation-script) -to find workflow arguments. +Your can generate a library by using the +[`generate_new_client.yaml` Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml). +This workflow runs `new-client.py` with your input arguments +and creates a pull request with the generated library. :warning: **IMPORTANT:** Not all the `new-client.py` arguments are available in the Github Action. @@ -18,7 +18,7 @@ Please refer to [this section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) for more arguments (it requires to setup a local environment). -The arguments currently supported by the workflow are: +The arguments currently supported by the Github Action are: - [API short Name (`api_shortname`)](#api-short-name) - [Proto path (`proto_path`)](#proto-path) - [Name pretty (`name_pretty`)](#name-pretty) @@ -57,13 +57,13 @@ the repository has "-SNAPSHOT" versions too. Some languages require a new request when a new version of a service is created, but Java manages all versions of the service as a single package, so the client library will automatically be updated to include new versions. -## Execute the workflow (either locally or via Github Action) +## Execute the Github Action You will run new-client.py script or the [Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) with the following parameters. These parameters will be available in the Cloud Drop link (a YAML file) included in the buganizer request. The example in this README uses AlloyDB's [Cloud Drop](https://github.com/googleapis/googleapis/blob/master/google/cloud/alloydb/v1/alloydb_v1.yaml) file as an example. -### API short name +### API short name (`api_shortname`) For convenience of the subsequent commands, define a variable for API short name. This value will be used by default to generate the following: @@ -79,7 +79,7 @@ Example: `alloydb` > In the instance that the `api_short_name` is already in use by an existing client library, you will need to determine a unique name. > See example under [Advanced Options](#Example with duplicate api_short_name). -### Proto path +### Proto path (`proto_path`) The script takes "proto path" parameter. This is the path from the internal `google3/third_party/googleapis/stable` root to the directory that contains versions (e.g., "v1" or "v2"). @@ -95,20 +95,20 @@ Therefore, the "proto path" value we supply to the command is `google/cloud/allo We will publish a single module for a service that includes all versions in this path. Once the service has been published once, any future additional versions will automatically be generated via OwlBot. -### Name pretty +### Name pretty (`name_pretty`) The corresponding value in the Cloud Drop page is `title`. Example: `AlloyDB API` -### Product Docs +### Product Docs (`product_docs`) The corresponding value in the Cloud Drop page is `documentation_uri`. The value must starts with "https://". Example: `https://cloud.google.com/alloydb/docs` -### REST Docs +### REST Docs (`rest_docs`) The corresponding value in the Cloud Drop page is `rest_reference_documentation_uri`. The value must starts with "https://". @@ -118,7 +118,7 @@ Example: `https://cloud.google.com/alloydb/docs/reference/rest` If they exist, add them as a flag to the python command below like: `--rest-docs="https://cloud.google.com/alloydb/docs/reference/rest" \` -### RPC Docs +### RPC Docs (`rpc_docs`) The corresponding value in the Cloud Drop page is `proto_reference_documentation_uri`. The value must starts with "https://". @@ -128,7 +128,7 @@ Example: `https://cloud.google.com/speech-to-text/docs/reference/rpc` If they exist, add them as a flag to the python command below like: `--rpc-docs="https://cloud.google.com/speech-to-text/docs/reference/rpc" \` -### API description +### API description (`api_description`) The corresponding value in the Cloud Drop page is `documentation.summary` or `documentation.overview`. If both of those fields are missing, take the description from the product page above. Use the first sentence to keep it concise. @@ -140,76 +140,41 @@ Example: commercial-grade applications. ``` +### Transport (`transport`) + +This is a label that represents the type of requests the library will make to +its corresponding service. It can be `grpc` (default), `http` or `both`. In +practice, it is mainly used to create a `.repo-metadata.json` file + +### Destination Name (`destination_name`) + +This variable represents the folder name to be created in the root of the +monorepo. For example, if it is `java-example-library`, then a new folder +with the generated library will be created in +`google-cloud-java/java-example/library` + +### Distribution Name (`distribution_name`) + +This variable determines the Maven coordinates of the generated library. It +defaults to `com.google.cloud:google-cloud-{api_shortname}`. This mainly affect +the values in the generated `pom.xml` files. ## Advanced Options +In case the steps above don't show you how to specify the desired options, you can +run the `new-client.py` script in your local evironment. For the explanation of the available parameters, run: `python3.9 generation/new_client/new-client.py generate --help`. -:warning: **Note: The advanced options other than `transport`, `destination-name` and -`distribution-name` cannot be specified in the [Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml), you will have to run -the script locally (refer to the Prerequisites (for local environment) section -below) if you need to specify any of these** - -``` -~/google-cloud-java$ python3.9 generation/new_client/new-client.py generate --help -/usr/local/google/home/suztomo/google-cloud-java/generation/new_client -Usage: new-client.py generate [OPTIONS] - -Options: - --api_shortname TEXT Name for the new directory name and - (default) artifact name [required] - --name-pretty TEXT The human-friendly name that appears in - README.md [required] - --product-docs TEXT Documentation URL that appears in README.md - [required] - --api-description TEXT Description that appears in README.md - [required] - --release-level [stable|preview] - A label that appears in repo-metadata.json. - The first library generation is always - 'preview'. [default: preview] - --transport [grpc|http|both] A label that appears in repo-metadata.json - [default: grpc] - --language TEXT [default: java] - --distribution-name TEXT Maven coordinates of the generated library. - By default it's com.google.cloud:google- - cloud- - --api-id TEXT The value of the apiid parameter used in - README.md It has link to https://console.clo - ud.google.com/flows/enableapi?apiid= - --requires-billing BOOLEAN Based on this value, README.md explains - whether billing setup is needed or not. - [default: True] - --destination-name TEXT The directory name of the new library. By - default it's java- - --proto-path TEXT Path to proto file from the root of the - googleapis repository to thedirectory that - contains the proto files (without the - version).For example, to generate the - library for 'google/maps/routing/v2', then - you specify this value as - 'google/maps/routing' [required] - --cloud-api BOOLEAN If true, the artifact ID of the library is - 'google-cloud-'; otherwise 'google-' - [default: True] - --group-id TEXT The group ID of the artifact when - distribution name is not set [default: - com.google.cloud] - --owlbot-image TEXT The owlbot container image used in - OwlBot.yaml [default: gcr.io/cloud-devrel- - public-resources/owlbot-java] - --library-type TEXT A label that appear in repo-metadata.json to - tell how the library is maintained or - generated [default: GAPIC_AUTO] - --googleapis-gen-url TEXT The URL of the repository that has generated - Java code from proto service definition - [default: - https://github.com/googleapis/googleapis- - gen.git] - --help Show this message and exit. -``` +:warning: **Note: The advanced options that don't appear in the previous section +cannot be specified in the +[Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) +. You will have to run the script locally (refer to the "Prerequisites +(for local environment)" section below for setup instructions) +if you need to specify any of these** + +### Special case example: Google Maps Sometimes, a library generation requires special handling for Maven coordinates or API ID, especially when the library is not From 26162d0a7fbec74a40bac00566847d763cd9af2e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 17 Jan 2024 16:06:22 +0000 Subject: [PATCH 47/51] fix "option" bias in doc --- generation/new_client/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 549dc5421908..5dae5ea14a61 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -26,9 +26,9 @@ The arguments currently supported by the Github Action are: - [REST Docs (`rest_docs`)](#rest-docs) - [RPC Docs (`rpc_docs`)](#rpc-docs) - [API description (`api_description`)](#api-description) -- [`transport`](#advanced-options) -- [`destination_name`](#advanced-options) -- [`distribution_name`](#advanced-options) +- [Transport (`transport`)](#transport) +- [Destination Name (`destination_name`)](#destination-name) +- [Distribution Name (`distribution_name`)](#distribution-name) ### Accessing the generated PR @@ -59,7 +59,9 @@ Some languages require a new request when a new version of a service is created, ## Execute the Github Action -You will run new-client.py script or the [Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) with the following parameters. +You in order to run +[Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) +, you need to specify a few parameters. These parameters will be available in the Cloud Drop link (a YAML file) included in the buganizer request. The example in this README uses AlloyDB's [Cloud Drop](https://github.com/googleapis/googleapis/blob/master/google/cloud/alloydb/v1/alloydb_v1.yaml) file as an example. From 9ef6e561a1fe1e738738c50be822c9217a8889f3 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 17 Jan 2024 16:09:55 +0000 Subject: [PATCH 48/51] remove links for arguments --- generation/new_client/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 5dae5ea14a61..65f6f837c507 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -19,16 +19,16 @@ Please refer to section](https://github.com/googleapis/google-cloud-java/blob/main/generation/new_client/README.md#advanced-options) for more arguments (it requires to setup a local environment). The arguments currently supported by the Github Action are: -- [API short Name (`api_shortname`)](#api-short-name) -- [Proto path (`proto_path`)](#proto-path) -- [Name pretty (`name_pretty`)](#name-pretty) -- [Product Docs (`product_docs`)](#product-docs) -- [REST Docs (`rest_docs`)](#rest-docs) -- [RPC Docs (`rpc_docs`)](#rpc-docs) -- [API description (`api_description`)](#api-description) -- [Transport (`transport`)](#transport) -- [Destination Name (`destination_name`)](#destination-name) -- [Distribution Name (`distribution_name`)](#distribution-name) +- API short Name (`api_shortname`) +- Proto path (`proto_path`) +- Name pretty (`name_pretty`) +- Product Docs (`product_docs`) +- REST Docs (`rest_docs`) +- RPC Docs (`rpc_docs`) +- API description (`api_description`) +- Transport (`transport`) +- Destination Name (`destination_name`) +- Distribution Name (`distribution_name`) ### Accessing the generated PR From f255a2b5d17676c07fae1a5490779acda301c176 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 17 Jan 2024 16:11:19 +0000 Subject: [PATCH 49/51] punctuation --- generation/new_client/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 65f6f837c507..4363ac313ed7 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -144,16 +144,16 @@ Example: ### Transport (`transport`) -This is a label that represents the type of requests the library will make to +This variable represents the type of requests the library will make to its corresponding service. It can be `grpc` (default), `http` or `both`. In -practice, it is mainly used to create a `.repo-metadata.json` file +practice, it is mainly used to create a `.repo-metadata.json` file. ### Destination Name (`destination_name`) This variable represents the folder name to be created in the root of the monorepo. For example, if it is `java-example-library`, then a new folder with the generated library will be created in -`google-cloud-java/java-example/library` +`google-cloud-java/java-example/library`. ### Distribution Name (`distribution_name`) From 99857c0d8a594aec590ac51e4db9e2cd771fc7e3 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 17 Jan 2024 16:14:27 +0000 Subject: [PATCH 50/51] remove redundant advanced options explanation --- generation/new_client/README.md | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index 4363ac313ed7..b9b26063bad0 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -165,17 +165,13 @@ the values in the generated `pom.xml` files. ## Advanced Options In case the steps above don't show you how to specify the desired options, you can -run the `new-client.py` script in your local evironment. +run the `new-client.py` script in your local evironment. The advanced options +not shown in the section above **cannot be specified in the Github Action**, +hence the need for a local run (refer to the "Prerequisites +(for local environment)" section). For the explanation of the available parameters, run: `python3.9 generation/new_client/new-client.py generate --help`. -:warning: **Note: The advanced options that don't appear in the previous section -cannot be specified in the -[Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client.yaml) -. You will have to run the script locally (refer to the "Prerequisites -(for local environment)" section below for setup instructions) -if you need to specify any of these** - ### Special case example: Google Maps Sometimes, a library generation requires special handling for @@ -223,12 +219,6 @@ Let's say that after some discussion, `maps-routing-gps` is selected as a suitab --destination-name="maps-routing-gps" ``` -# Principles - -The script should finish creating a pull request even when the newly created -module fails to compile. This gives the user flexibility to fix things in the -created pull request. - ## Prerequisites (for local environment) This section is only needed for the first _local_ run of this script. If it's already From 7beb584c098e643763a43b3469d21495b20cf84c Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 17 Jan 2024 18:53:02 +0000 Subject: [PATCH 51/51] restore principles section --- generation/new_client/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/generation/new_client/README.md b/generation/new_client/README.md index b9b26063bad0..93248d203af7 100644 --- a/generation/new_client/README.md +++ b/generation/new_client/README.md @@ -5,6 +5,7 @@ google-cloud-java monorepo. **This tool is for repository maintainers only. Not for library users.** + ## Run via Github Action Your can generate a library by using the @@ -357,6 +358,12 @@ Create a pull request from the change. In the description, record the `python3.9 generation/new_client/new-client.py generate ...` command you ran above. +# Principles + +The script should finish creating a pull request even when the newly created +module fails to compile. This gives the user flexibility to fix things in the +created pull request. + # Common Errors ## Unable to clone googleapis-gen