Consume the AWS API models from Maven instead of checking it out from git

adwsingh · adwsingh · commit 3c800f818f34 · 2026-04-04T17:36:55.000-07:00
diff --git a/.github/workflows/codegen-compatibility.yml b/.github/workflows/codegen-compatibility.yml
@@ -19,12 +19,6 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - name: Checkout api-models-aws
-        uses: actions/checkout@v6
-        with:
-          repository: aws/api-models-aws
-          path: api-models-aws
-
       - name: Set up JDK 21
         uses: actions/setup-java@v5
         with:
@@ -46,10 +40,8 @@ jobs:
             caches
 
       - name: Run codegen compatibility tests
-        env:
-          API_MODELS_AWS_DIR: ${{ github.workspace }}/api-models-aws
         run: |
-          ./gradlew :codegen:codegen-plugin:integ --tests "*.AwsModelCodegenCompilationTest" --stacktrace
+          ./gradlew :codegen:codegen-plugin:integ -PawsModelsTests --tests "*.AwsModelCodegenCompilationTest" --stacktrace
 
       - uses: actions/upload-artifact@v7
         if: failure()
diff --git a/.github/workflows/endpoint-compatibility.yml b/.github/workflows/endpoint-compatibility.yml
@@ -19,12 +19,6 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - name: Checkout api-models-aws
-        uses: actions/checkout@v6
-        with:
-          repository: aws/api-models-aws
-          path: api-models-aws
-
       - name: Set up JDK 21
         uses: actions/setup-java@v5
         with:
@@ -46,10 +40,8 @@ jobs:
             caches
 
       - name: Run endpoint ruleset compatibility tests
-        env:
-          API_MODELS_AWS_DIR: ${{ github.workspace }}/api-models-aws
         run: |
-          ./gradlew :client:client-rulesengine:integ --tests "*.AwsEndpointRulesetTest" --stacktrace
+          ./gradlew :client:client-rulesengine:integ -PawsModelsTests --tests "*.AwsEndpointRulesetTest" --stacktrace
 
       - uses: actions/upload-artifact@v7
         if: failure()
diff --git a/buildSrc/src/main/kotlin/smithy-java.integ-test-conventions.gradle.kts b/buildSrc/src/main/kotlin/smithy-java.integ-test-conventions.gradle.kts
@@ -1,4 +1,7 @@
 
+import javax.xml.parsers.DocumentBuilderFactory
+import org.w3c.dom.Element
+
 configure<SourceSetContainer> {
     val main by getting
     val test by getting
@@ -16,3 +19,54 @@ tasks.register<Test>("integ") {
     testClassesDirs = project.the<SourceSetContainer>()["it"].output.classesDirs
     classpath = project.the<SourceSetContainer>()["it"].runtimeClasspath
 }
+
+// Extension for configuring integration test behavior.
+// Projects can opt in to AWS API model tests by adding:
+//   configureIntegTests { awsModels = true }
+// Models are only resolved when the `-PawsModelsTests` Gradle property is also set.
+interface IntegTestExtension {
+    var awsModelTests: Boolean
+}
+
+val configureIntegTests = extensions.create<IntegTestExtension>("configureIntegTests").apply {
+    awsModelTests = false
+}
+
+afterEvaluate {
+    if (!configureIntegTests.awsModelTests || !project.hasProperty("awsModelsTests")) {
+        return@afterEvaluate
+    }
+
+    val libs = extensions.getByType<VersionCatalogsExtension>().named("libs")
+    val bom = libs.findLibrary("aws-api-models-bom").get().get()
+    val bomCoords = "${bom.module.group}:${bom.module.name}:${bom.version}"
+
+    val bomFile = configurations.detachedConfiguration(
+        dependencies.platform(bomCoords),
+        dependencies.create("$bomCoords@pom")
+    ).apply {
+        isTransitive = false
+    }.files.single { it.extension == "pom" }
+
+    val doc = DocumentBuilderFactory.newInstance()
+        .newDocumentBuilder()
+        .parse(bomFile)
+
+    val deps = doc.getElementsByTagName("dependency")
+    dependencies.add("itImplementation", dependencies.platform(bomCoords))
+
+    for (i in 0 until deps.length) {
+        val dep = deps.item(i) as? Element ?: continue
+        val g = dep.getElementsByTagName("groupId").item(0)?.textContent ?: continue
+        val a = dep.getElementsByTagName("artifactId").item(0)?.textContent ?: continue
+
+        if (g == "software.amazon.api.models") {
+            dependencies.add("itImplementation", "$g:$a")
+        }
+    }
+
+    tasks.named<Test>("integ") {
+        systemProperty("awsModelsTests", "true")
+        maxHeapSize = "4g"
+    }
+}
diff --git a/client/client-rulesengine/build.gradle.kts b/client/client-rulesengine/build.gradle.kts
@@ -16,3 +16,7 @@ dependencies {
     testImplementation(project(":client:dynamic-client"))
     testImplementation(project(":aws:client:aws-client-rulesengine"))
 }
+
+configureIntegTests {
+    awsModels = true
+}
diff --git a/client/client-rulesengine/src/it/java/software/amazon/smithy/java/client/rulesengine/AwsEndpointRulesetTest.java b/client/client-rulesengine/src/it/java/software/amazon/smithy/java/client/rulesengine/AwsEndpointRulesetTest.java
@@ -9,19 +9,19 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.net.URL;
+import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Stream;
-import org.junit.jupiter.api.condition.EnabledIf;
+import org.junit.jupiter.api.Named;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import software.amazon.smithy.java.rulesengine.Bytecode;
 import software.amazon.smithy.java.rulesengine.RulesEngineBuilder;
 import software.amazon.smithy.model.Model;
 import software.amazon.smithy.model.loader.ModelAssembler;
+import software.amazon.smithy.model.loader.ModelDiscovery;
 import software.amazon.smithy.model.shapes.ServiceShape;
 import software.amazon.smithy.rulesengine.language.EndpointRuleSet;
 import software.amazon.smithy.rulesengine.language.evaluation.TestEvaluator;
@@ -38,40 +38,36 @@
  * evaluated against its endpoint test cases, round-tripped through serialization,
  * and disassembled without errors.
  *
- * <p>Set the {@code API_MODELS_AWS_DIR} environment variable to the root of a local
- * checkout of api-models-aws to enable this test.
+ * <p>AWS service models are discovered on the classpath via
+ * {@link ModelDiscovery#findModels()}.
  */
+@EnabledIfSystemProperty(named = "awsModelsTests", matches = "true")
 class AwsEndpointRulesetTest {
 
-    private static Path getModelsDir() {
-        var dir = System.getenv("API_MODELS_AWS_DIR");
-        return dir != null ? Paths.get(dir, "models") : null;
-    }
-
-    static boolean modelsAvailable() {
-        var dir = getModelsDir();
-        return dir != null && Files.isDirectory(dir);
+    static Stream<Named<URL>> awsModels() {
+        return ModelDiscovery.findModels()
+                .stream()
+                .filter(url -> url.toString().endsWith(".json"))
+                .map(url -> Named.of(artifactName(url), url))
+                .sorted(Comparator.comparing(Named::getName));
     }
 
-    static Stream<Path> awsModels() throws IOException {
-        var modelsDir = getModelsDir();
-        try (var paths = Files.find(modelsDir,
-                4,
-                (p, a) -> p.toString().endsWith(".json")
-                        && p.getParent().getParent().getFileName().toString().equals("service"))) {
-            return paths.sorted().toList().stream();
-        }
+    private static String artifactName(URL modelUrl) {
+        String urlStr = modelUrl.toString();
+        int bangIdx = urlStr.indexOf("!/");
+        String jarPath = urlStr.substring(0, bangIdx);
+        String jarName = jarPath.substring(jarPath.lastIndexOf('/') + 1);
+        return jarName.replaceFirst("-\\d[\\d.]*\\.jar$", "");
     }
 
-    @EnabledIf("modelsAvailable")
     @ParameterizedTest(name = "{0}")
     @MethodSource("awsModels")
-    void compileEvaluateAndRoundTrip(Path modelPath) {
+    void compileEvaluateAndRoundTrip(URL modelUrl) {
         // Load the model
         Model model = Model.assembler()
-                .addImport(modelPath)
+                .addImport(modelUrl)
                 .putProperty(ModelAssembler.ALLOW_UNKNOWN_TRAITS, true)
-                .discoverModels()
+                .disableValidation()
                 .assemble()
                 .unwrap();
 
diff --git a/codegen/codegen-plugin/build.gradle.kts b/codegen/codegen-plugin/build.gradle.kts
@@ -92,13 +92,6 @@ tasks.test {
     failOnNoDiscoveredTests = false
 }
 
-// If API_MODELS_AWS_DIR is set, add it as a task input so Gradle doesn't cache results.
-System.getenv("API_MODELS_AWS_DIR")?.let { modelsDir ->
-    val dir = file(modelsDir)
-    if (dir.isDirectory) {
-        tasks.named<Test>("integ") {
-            inputs.dir(dir).withPathSensitivity(PathSensitivity.RELATIVE)
-            maxHeapSize = "4g"
-        }
-    }
+configureIntegTests {
+    awsModels = true
 }
diff --git a/codegen/codegen-plugin/src/it/java/software/amazon/smithy/java/codegen/AwsModelCodegenCompilationTest.java b/codegen/codegen-plugin/src/it/java/software/amazon/smithy/java/codegen/AwsModelCodegenCompilationTest.java
@@ -13,11 +13,12 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.URI;
+import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -33,7 +34,7 @@
 import javax.tools.StandardJavaFileManager;
 import javax.tools.ToolProvider;
 import org.junit.jupiter.api.Named;
-import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
 import org.junit.jupiter.api.parallel.Execution;
 import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -42,51 +43,52 @@
 import software.amazon.smithy.build.PluginContext;
 import software.amazon.smithy.model.Model;
 import software.amazon.smithy.model.loader.ModelAssembler;
+import software.amazon.smithy.model.loader.ModelDiscovery;
 import software.amazon.smithy.model.node.ArrayNode;
 import software.amazon.smithy.model.node.ObjectNode;
 import software.amazon.smithy.model.shapes.ServiceShape;
 
 /**
  * Tests that Java code generation produces compilable code for all AWS service models.
  *
- * <p>Set the {@code API_MODELS_AWS_DIR} environment variable to the root of a local
- * checkout of {@code aws/api-models-aws} to enable this test.
+ * <p>AWS service models are discovered on the classpath via
+ * {@link ModelDiscovery#findModels()}.
  */
-@EnabledIfEnvironmentVariable(named = "API_MODELS_AWS_DIR", matches = ".*")
+@EnabledIfSystemProperty(named = "awsModelsTests", matches = "true")
 class AwsModelCodegenCompilationTest {
 
     private static final Set<String> IGNORED_SDK_IDS = Set.of(
             "timestream-write",
             "timestream-query",
-            "clouddirectory"
+            "clouddirectory");
 
-    );
-
-    private static Path getModelsDir() {
-        var dir = System.getenv("API_MODELS_AWS_DIR");
-        return Paths.get(dir, "models");
+    static Stream<Named<URL>> awsModels() {
+        return ModelDiscovery.findModels()
+                .stream()
+                .filter(url -> url.toString().endsWith(".json"))
+                .map(url -> Named.of(artifactName(url), url))
+                .sorted(Comparator.comparing(Named::getName));
     }
 
-    static Stream<Named<Path>> awsModels() throws IOException {
-        var modelsDir = getModelsDir();
-        return Files.find(modelsDir,
-                4,
-                (p, a) -> p.toString().endsWith(".json")
-                        && p.getParent().getParent().getFileName().toString().equals("service"))
-                .map(p -> Named.of(sdkId(modelsDir, p), p));
+    private static String artifactName(URL modelUrl) {
+        String urlStr = modelUrl.toString();
+        int bangIdx = urlStr.indexOf("!/");
+        String jarPath = urlStr.substring(0, bangIdx);
+        String jarName = jarPath.substring(jarPath.lastIndexOf('/') + 1);
+        return jarName.replaceFirst("-\\d[\\d.]*\\.jar$", "");
     }
 
     @ParameterizedTest(name = "client: {0}")
     @MethodSource("awsModels")
     @Execution(ExecutionMode.CONCURRENT)
-    void compileGeneratedCode(Path modelPath) {
-        generateAndCompile(modelPath, "client", "server");
+    void compileGeneratedCode(URL modelUrl) {
+        generateAndCompile(modelUrl, "client", "server");
     }
 
-    private void generateAndCompile(Path modelPath, String... modes) {
+    private void generateAndCompile(URL modelUrl, String... modes) {
         // 1. Load model
         Model model = Model.assembler()
-                .addImport(modelPath)
+                .addImport(modelUrl)
                 .putProperty(ModelAssembler.ALLOW_UNKNOWN_TRAITS, true)
                 .disableValidation()
                 .assemble()
@@ -149,7 +151,7 @@ private void generateAndCompile(Path modelPath, String... modes) {
                 }
             }
         } catch (Throwable t) {
-            if (IGNORED_SDK_IDS.contains(sdkId(getModelsDir(), modelPath))) {
+            if (IGNORED_SDK_IDS.contains(artifactName(modelUrl))) {
                 abort("Known failure for " + service.getId() + ": " + t.getMessage());
             }
             sneakyThrow(t);
@@ -177,10 +179,6 @@ private static Path dumpGeneratedSources(String serviceId, List<JavaFileObject>
         }
     }
 
-    private static String sdkId(Path modelsDir, Path modelPath) {
-        return modelsDir.relativize(modelPath).getName(0).toString();
-    }
-
     private static String sanitize(String name) {
         return name.toLowerCase().replaceAll("[^a-z0-9]", "");
     }
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -22,6 +22,7 @@ jazzer = "0.30.0"
 json-schema-validator = "3.0.0"
 opentelemetry = "1.60.1"
 jspecify = "1.0.0"
+aws-api-models = "1.0.204"
 
 [libraries]
 smithy-model = { module = "software.amazon.smithy:smithy-model", version.ref = "smithy" }
@@ -73,6 +74,7 @@ jazzer-junit = { module = "com.code-intelligence:jazzer-junit", version.ref = "j
 jazzer-api = { module = "com.code-intelligence:jazzer-api", version.ref = "jazzer" }
 json-schema-validator = { module = "com.networknt:json-schema-validator", version.ref = "json-schema-validator"}
 jspecify = { module = "org.jspecify:jspecify", version.ref = "jspecify" }
+aws-api-models-bom = { module = "software.amazon.api.models:all", version.ref = "aws-api-models" }
 
 # plugin artifacts for buildsrc plugins
 test-logger-plugin = { module = "com.adarshr:gradle-test-logger-plugin", version.ref = "test-logger-plugin" }

Original file line number	Diff line number	Diff line change
`@@ -16,3 +16,7 @@ dependencies {`
`16`	`16`	`testImplementation(project(":client:dynamic-client"))`
`17`	`17`	`testImplementation(project(":aws:client:aws-client-rulesengine"))`
`18`	`18`	`}`
	`19`	`+`
	`20`	`+configureIntegTests {`
	`21`	`+ awsModels = true`
	`22`	`+}`