From 65ebac3fddcb0211d4eb058f7c7cc4c506fc5ee8 Mon Sep 17 00:00:00 2001 From: Adwait Kumar Singh Date: Sat, 4 Apr 2026 17:26:44 -0700 Subject: [PATCH] Consume the AWS API models from Maven instead of checking it out from git --- .github/workflows/codegen-compatibility.yml | 10 +--- .github/workflows/endpoint-compatibility.yml | 10 +--- ...thy-java.integ-test-conventions.gradle.kts | 54 +++++++++++++++++++ client/client-rulesengine/build.gradle.kts | 4 ++ .../rulesengine/AwsEndpointRulesetTest.java | 50 ++++++++--------- codegen/codegen-plugin/build.gradle.kts | 11 +--- .../AwsModelCodegenCompilationTest.java | 52 +++++++++--------- gradle/libs.versions.toml | 2 + 8 files changed, 112 insertions(+), 81 deletions(-) diff --git a/.github/workflows/codegen-compatibility.yml b/.github/workflows/codegen-compatibility.yml index dd3179690..4ecea218b 100644 --- a/.github/workflows/codegen-compatibility.yml +++ b/.github/workflows/codegen-compatibility.yml @@ -19,12 +19,6 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Checkout api-models-aws - uses: actions/checkout@v6 - with: - repository: aws/api-models-aws - path: api-models-aws - - name: Set up JDK 21 uses: actions/setup-java@v5 with: @@ -46,10 +40,8 @@ jobs: caches - name: Run codegen compatibility tests - env: - API_MODELS_AWS_DIR: ${{ github.workspace }}/api-models-aws run: | - ./gradlew :codegen:codegen-plugin:integ --tests "*.AwsModelCodegenCompilationTest" --stacktrace + ./gradlew :codegen:codegen-plugin:integ -PawsModelsTests --tests "*.AwsModelCodegenCompilationTest" --stacktrace - uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/endpoint-compatibility.yml b/.github/workflows/endpoint-compatibility.yml index a7f2fec78..1df540c49 100644 --- a/.github/workflows/endpoint-compatibility.yml +++ b/.github/workflows/endpoint-compatibility.yml @@ -19,12 +19,6 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Checkout api-models-aws - uses: actions/checkout@v6 - with: - repository: aws/api-models-aws - path: api-models-aws - - name: Set up JDK 21 uses: actions/setup-java@v5 with: @@ -46,10 +40,8 @@ jobs: caches - name: Run endpoint ruleset compatibility tests - env: - API_MODELS_AWS_DIR: ${{ github.workspace }}/api-models-aws run: | - ./gradlew :client:client-rulesengine:integ --tests "*.AwsEndpointRulesetTest" --stacktrace + ./gradlew :client:client-rulesengine:integ -PawsModelsTests --tests "*.AwsEndpointRulesetTest" --stacktrace - uses: actions/upload-artifact@v7 if: failure() diff --git a/buildSrc/src/main/kotlin/smithy-java.integ-test-conventions.gradle.kts b/buildSrc/src/main/kotlin/smithy-java.integ-test-conventions.gradle.kts index e841cbc2d..402bf52ab 100644 --- a/buildSrc/src/main/kotlin/smithy-java.integ-test-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/smithy-java.integ-test-conventions.gradle.kts @@ -1,4 +1,7 @@ +import javax.xml.parsers.DocumentBuilderFactory +import org.w3c.dom.Element + configure { val main by getting val test by getting @@ -16,3 +19,54 @@ tasks.register("integ") { testClassesDirs = project.the()["it"].output.classesDirs classpath = project.the()["it"].runtimeClasspath } + +// Extension for configuring integration test behavior. +// Projects can opt in to AWS API model tests by adding: +// configureIntegTests { awsModelTests = true } +// Models are only resolved when the `-PawsModelTests` Gradle property is also set. +interface IntegTestExtension { + var awsModelTests: Boolean +} + +val configureIntegTests = extensions.create("configureIntegTests").apply { + awsModelTests = false +} + +afterEvaluate { + if (!configureIntegTests.awsModelTests || !project.hasProperty("awsModelsTests")) { + return@afterEvaluate + } + + val libs = extensions.getByType().named("libs") + val bom = libs.findLibrary("aws-api-models-bom").get().get() + val bomCoords = "${bom.module.group}:${bom.module.name}:${bom.version}" + + val bomFile = configurations.detachedConfiguration( + dependencies.platform(bomCoords), + dependencies.create("$bomCoords@pom") + ).apply { + isTransitive = false + }.files.single { it.extension == "pom" } + + val doc = DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(bomFile) + + val deps = doc.getElementsByTagName("dependency") + dependencies.add("itImplementation", dependencies.platform(bomCoords)) + + for (i in 0 until deps.length) { + val dep = deps.item(i) as? Element ?: continue + val g = dep.getElementsByTagName("groupId").item(0)?.textContent ?: continue + val a = dep.getElementsByTagName("artifactId").item(0)?.textContent ?: continue + + if (g == "software.amazon.api.models") { + dependencies.add("itImplementation", "$g:$a") + } + } + + tasks.named("integ") { + systemProperty("awsModelsTests", "true") + maxHeapSize = "4g" + } +} diff --git a/client/client-rulesengine/build.gradle.kts b/client/client-rulesengine/build.gradle.kts index 31cbd2b3f..70877bc03 100644 --- a/client/client-rulesengine/build.gradle.kts +++ b/client/client-rulesengine/build.gradle.kts @@ -16,3 +16,7 @@ dependencies { testImplementation(project(":client:dynamic-client")) testImplementation(project(":aws:client:aws-client-rulesengine")) } + +configureIntegTests { + awsModelTests = true +} diff --git a/client/client-rulesengine/src/it/java/software/amazon/smithy/java/client/rulesengine/AwsEndpointRulesetTest.java b/client/client-rulesengine/src/it/java/software/amazon/smithy/java/client/rulesengine/AwsEndpointRulesetTest.java index e6e2b3064..a3310ced0 100644 --- a/client/client-rulesengine/src/it/java/software/amazon/smithy/java/client/rulesengine/AwsEndpointRulesetTest.java +++ b/client/client-rulesengine/src/it/java/software/amazon/smithy/java/client/rulesengine/AwsEndpointRulesetTest.java @@ -9,19 +9,19 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.net.URL; +import java.util.Comparator; import java.util.List; import java.util.stream.Stream; -import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.jupiter.api.Named; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import software.amazon.smithy.java.rulesengine.Bytecode; import software.amazon.smithy.java.rulesengine.RulesEngineBuilder; import software.amazon.smithy.model.Model; import software.amazon.smithy.model.loader.ModelAssembler; +import software.amazon.smithy.model.loader.ModelDiscovery; import software.amazon.smithy.model.shapes.ServiceShape; import software.amazon.smithy.rulesengine.language.EndpointRuleSet; import software.amazon.smithy.rulesengine.language.evaluation.TestEvaluator; @@ -38,40 +38,36 @@ * evaluated against its endpoint test cases, round-tripped through serialization, * and disassembled without errors. * - *

Set the {@code API_MODELS_AWS_DIR} environment variable to the root of a local - * checkout of api-models-aws to enable this test. + *

AWS service models are discovered on the classpath via + * {@link ModelDiscovery#findModels()}. */ +@EnabledIfSystemProperty(named = "awsModelsTests", matches = "true") class AwsEndpointRulesetTest { - private static Path getModelsDir() { - var dir = System.getenv("API_MODELS_AWS_DIR"); - return dir != null ? Paths.get(dir, "models") : null; - } - - static boolean modelsAvailable() { - var dir = getModelsDir(); - return dir != null && Files.isDirectory(dir); + static Stream> awsModels() { + return ModelDiscovery.findModels() + .stream() + .filter(url -> url.toString().endsWith(".json")) + .map(url -> Named.of(artifactName(url), url)) + .sorted(Comparator.comparing(Named::getName)); } - static Stream awsModels() throws IOException { - var modelsDir = getModelsDir(); - try (var paths = Files.find(modelsDir, - 4, - (p, a) -> p.toString().endsWith(".json") - && p.getParent().getParent().getFileName().toString().equals("service"))) { - return paths.sorted().toList().stream(); - } + private static String artifactName(URL modelUrl) { + String urlStr = modelUrl.toString(); + int bangIdx = urlStr.indexOf("!/"); + String jarPath = urlStr.substring(0, bangIdx); + String jarName = jarPath.substring(jarPath.lastIndexOf('/') + 1); + return jarName.replaceFirst("-\\d[\\d.]*\\.jar$", ""); } - @EnabledIf("modelsAvailable") @ParameterizedTest(name = "{0}") @MethodSource("awsModels") - void compileEvaluateAndRoundTrip(Path modelPath) { + void compileEvaluateAndRoundTrip(URL modelUrl) { // Load the model Model model = Model.assembler() - .addImport(modelPath) + .addImport(modelUrl) .putProperty(ModelAssembler.ALLOW_UNKNOWN_TRAITS, true) - .discoverModels() + .disableValidation() .assemble() .unwrap(); diff --git a/codegen/codegen-plugin/build.gradle.kts b/codegen/codegen-plugin/build.gradle.kts index 573bed6c3..f798fa41b 100644 --- a/codegen/codegen-plugin/build.gradle.kts +++ b/codegen/codegen-plugin/build.gradle.kts @@ -92,13 +92,6 @@ tasks.test { failOnNoDiscoveredTests = false } -// If API_MODELS_AWS_DIR is set, add it as a task input so Gradle doesn't cache results. -System.getenv("API_MODELS_AWS_DIR")?.let { modelsDir -> - val dir = file(modelsDir) - if (dir.isDirectory) { - tasks.named("integ") { - inputs.dir(dir).withPathSensitivity(PathSensitivity.RELATIVE) - maxHeapSize = "4g" - } - } +configureIntegTests { + awsModelTests = true } diff --git a/codegen/codegen-plugin/src/it/java/software/amazon/smithy/java/codegen/AwsModelCodegenCompilationTest.java b/codegen/codegen-plugin/src/it/java/software/amazon/smithy/java/codegen/AwsModelCodegenCompilationTest.java index 1adb1fb4d..310b623b6 100644 --- a/codegen/codegen-plugin/src/it/java/software/amazon/smithy/java/codegen/AwsModelCodegenCompilationTest.java +++ b/codegen/codegen-plugin/src/it/java/software/amazon/smithy/java/codegen/AwsModelCodegenCompilationTest.java @@ -13,11 +13,12 @@ import java.io.IOException; import java.io.OutputStream; import java.net.URI; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -33,7 +34,7 @@ import javax.tools.StandardJavaFileManager; import javax.tools.ToolProvider; import org.junit.jupiter.api.Named; -import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; @@ -42,6 +43,7 @@ import software.amazon.smithy.build.PluginContext; import software.amazon.smithy.model.Model; import software.amazon.smithy.model.loader.ModelAssembler; +import software.amazon.smithy.model.loader.ModelDiscovery; import software.amazon.smithy.model.node.ArrayNode; import software.amazon.smithy.model.node.ObjectNode; import software.amazon.smithy.model.shapes.ServiceShape; @@ -49,44 +51,44 @@ /** * Tests that Java code generation produces compilable code for all AWS service models. * - *

Set the {@code API_MODELS_AWS_DIR} environment variable to the root of a local - * checkout of {@code aws/api-models-aws} to enable this test. + *

AWS service models are discovered on the classpath via + * {@link ModelDiscovery#findModels()}. */ -@EnabledIfEnvironmentVariable(named = "API_MODELS_AWS_DIR", matches = ".*") +@EnabledIfSystemProperty(named = "awsModelsTests", matches = "true") class AwsModelCodegenCompilationTest { private static final Set IGNORED_SDK_IDS = Set.of( "timestream-write", "timestream-query", - "clouddirectory" + "clouddirectory"); - ); - - private static Path getModelsDir() { - var dir = System.getenv("API_MODELS_AWS_DIR"); - return Paths.get(dir, "models"); + static Stream> awsModels() { + return ModelDiscovery.findModels() + .stream() + .filter(url -> url.toString().endsWith(".json")) + .map(url -> Named.of(artifactName(url), url)) + .sorted(Comparator.comparing(Named::getName)); } - static Stream> awsModels() throws IOException { - var modelsDir = getModelsDir(); - return Files.find(modelsDir, - 4, - (p, a) -> p.toString().endsWith(".json") - && p.getParent().getParent().getFileName().toString().equals("service")) - .map(p -> Named.of(sdkId(modelsDir, p), p)); + private static String artifactName(URL modelUrl) { + String urlStr = modelUrl.toString(); + int bangIdx = urlStr.indexOf("!/"); + String jarPath = urlStr.substring(0, bangIdx); + String jarName = jarPath.substring(jarPath.lastIndexOf('/') + 1); + return jarName.replaceFirst("-\\d[\\d.]*\\.jar$", ""); } @ParameterizedTest(name = "client: {0}") @MethodSource("awsModels") @Execution(ExecutionMode.CONCURRENT) - void compileGeneratedCode(Path modelPath) { - generateAndCompile(modelPath, "client", "server"); + void compileGeneratedCode(URL modelUrl) { + generateAndCompile(modelUrl, "client", "server"); } - private void generateAndCompile(Path modelPath, String... modes) { + private void generateAndCompile(URL modelUrl, String... modes) { // 1. Load model Model model = Model.assembler() - .addImport(modelPath) + .addImport(modelUrl) .putProperty(ModelAssembler.ALLOW_UNKNOWN_TRAITS, true) .disableValidation() .assemble() @@ -149,7 +151,7 @@ private void generateAndCompile(Path modelPath, String... modes) { } } } catch (Throwable t) { - if (IGNORED_SDK_IDS.contains(sdkId(getModelsDir(), modelPath))) { + if (IGNORED_SDK_IDS.contains(artifactName(modelUrl))) { abort("Known failure for " + service.getId() + ": " + t.getMessage()); } sneakyThrow(t); @@ -177,10 +179,6 @@ private static Path dumpGeneratedSources(String serviceId, List } } - private static String sdkId(Path modelsDir, Path modelPath) { - return modelsDir.relativize(modelPath).getName(0).toString(); - } - private static String sanitize(String name) { return name.toLowerCase().replaceAll("[^a-z0-9]", ""); } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 84e2cf511..ed17e6f71 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -22,6 +22,7 @@ jazzer = "0.30.0" json-schema-validator = "3.0.0" opentelemetry = "1.60.1" jspecify = "1.0.0" +aws-api-models = "1.0.204" [libraries] smithy-model = { module = "software.amazon.smithy:smithy-model", version.ref = "smithy" } @@ -73,6 +74,7 @@ jazzer-junit = { module = "com.code-intelligence:jazzer-junit", version.ref = "j jazzer-api = { module = "com.code-intelligence:jazzer-api", version.ref = "jazzer" } json-schema-validator = { module = "com.networknt:json-schema-validator", version.ref = "json-schema-validator"} jspecify = { module = "org.jspecify:jspecify", version.ref = "jspecify" } +aws-api-models-bom = { module = "software.amazon.api.models:all", version.ref = "aws-api-models" } # plugin artifacts for buildsrc plugins test-logger-plugin = { module = "com.adarshr:gradle-test-logger-plugin", version.ref = "test-logger-plugin" }