Merge pull request #142 from artpdr/ft-ap-lgbm-arm64

Add support for ARM64 on LGBM

Author: artpdr

.github/workflows/build.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - run: |
@@ -30,20 +30,29 @@ jobs:
           git tag --list
 
       - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          java-version: '8'
+          distribution: 'zulu'
 
       - name: Cache Maven packages
-        uses: actions/cache@v2.1.4
+        uses: actions/cache@v3.3.2
         with:
           path: ~/.m2
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-m2
 
+      - uses: docker/setup-qemu-action@v2
+        with:
+          platforms: "arm64"
+
       - name: Build and test
         run: mvn test -B
 
+      # The skipped tests is because h2o-xgboost isn't ready to use on arm64
+      - name: Test on arm64
+        run: docker run --rm --platform=arm64 -t -v ~/.m2:/root/.m2 -v $(pwd):/feedzai-openml-java maven:3.8-openjdk-8-slim /bin/bash -c 'apt update && apt install -y --no-install-recommends git && git config --global --add safe.directory /feedzai-openml-java && git config --global --add safe.directory /feedzai-openml-java/openml-lightgbm/lightgbm-builder/make-lightgbm && cd /feedzai-openml-java && mvn test -B -DfailIfNoTests=false -Dtest=!ClassifyUnknownCategoryTest#test,!H2OModelProviderTrainTest#trainModelsForAllAlgorithms'
+
       - uses: codecov/codecov-action@v1
         with:
           fail_ci_if_error: true

.github/workflows/deploy.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - run: |
@@ -24,13 +24,18 @@ jobs:
           git tag --list
 
       - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          java-version: '8'
+          distribution: 'zulu'
           server-id: ossrh
           server-username: MAVEN_USERNAME
           server-password: MAVEN_PASSWORD
 
+      - uses: docker/setup-qemu-action@v2
+        with:
+          platforms: "arm64"
+
       - name: Publish maven package
         run: mvn deploy -B -Prelease --batch-mode -DskipTests=true
         env:

README.md

@@ -6,12 +6,20 @@
 Implementations of the Feedzai OpenML API to allow support for machine
 learning models in Java. 
 
+All models may be used in nodes with AMD64 and ARM64 cpu architectures
+except for H2O-xgboost that can be used only with AMD64 cpu architectures.
+
 ## Building
 This is a Maven project which you can build using the following command:
 ```bash
 mvn clean install
 ```
 
+If the build fails compiling for ARM64 you may need to run:
+```bash
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+```
+
 ## Releasing
 
 For all releases, as the hotfix branch is ready all that's needed to actually release is to create an annotated tag pointing to the hotfix branch head.

openml-lightgbm/lightgbm-builder/make-lightgbm

@@ -43,8 +43,8 @@
                 <directory>make-lightgbm/build/</directory>
                 <filtering>false</filtering>
                 <includes>
-                    <include>*.so</include>
-                    <include>*.so.1.0.0</include>
+                    <include>**/*.so</include>
+                    <include>**/*.so.1.0.0</include>
                 </includes>
             </resource>
             <resource>

openml-lightgbm/lightgbm-builder/pom.xml

@@ -0,0 +1,35 @@
+/*
+ * The copyright of this file belongs to Feedzai. The file cannot be
+ * reproduced in whole or in part, stored in a retrieval system,
+ * transmitted in any form, or by any means electronic, mechanical,
+ * photocopying, or otherwise, without the prior permission of the owner.
+ *
+ * © 2023 Feedzai, Strictly Confidential
+ */
+package com.feedzai.openml.provider.lightgbm;
+
+/**
+ * Enum that represents the cpu architecture where code is running and consequent lgbm native libs locations.
+ */
+public enum CpuArchitecture {
+    AARCH64("arm64"),
+    AMD64("amd64");
+
+    /**
+     * This is the name of the folder where the lightgbm native libraries are.
+     */
+    private final String lgbmNativeLibsFolder;
+
+    CpuArchitecture(final String lgbmNativeLibsFolder){
+        this.lgbmNativeLibsFolder = lgbmNativeLibsFolder;
+    }
+
+    /**
+     * Gets the native libraries folder name according to the cpu architecture.
+     *
+     * @return the native libraries folder name according to the cpu architecture.
+     */
+    public String getLgbmNativeLibsFolder() {
+        return lgbmNativeLibsFolder;
+    }
+}

openml-lightgbm/lightgbm-provider/src/main/java/com/feedzai/openml/provider/lightgbm/CpuArchitecture.java

@@ -21,6 +21,9 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 import static com.google.common.io.ByteStreams.copy;
 
@@ -56,30 +59,45 @@ public class LightGBMUtils {
     static public void loadLibs() {
 
         if (!libsLoaded) {
+            final CpuArchitecture cpuArchitecture = getCpuArchitecture(System.getProperty("os.arch"));
             try {
-                loadSharedLibraryFromJar("libgomp.so.1.0.0");
-                loadSharedLibraryFromJar("lib_lightgbm.so");
-                loadSharedLibraryFromJar("lib_lightgbm_swig.so");
-            } catch (final IOException e) {
-                throw new RuntimeException("Failed to load LightGBM shared libraries from jar.", e);
+                loadSharedLibraryFromJar("libgomp.so.1.0.0", cpuArchitecture);
+                loadSharedLibraryFromJar("lib_lightgbm.so", cpuArchitecture);
+                loadSharedLibraryFromJar("lib_lightgbm_swig.so", cpuArchitecture);
+            } catch (final IOException ex) {
+                throw new RuntimeException("Failed to load LightGBM shared libraries from jar.", ex);
             }
 
             logger.info("Loaded LightGBM libs.");
             libsLoaded = true;
         }
     }
 
+    static CpuArchitecture getCpuArchitecture(final String cpuArchName) {
+        try {
+            return CpuArchitecture.valueOf(cpuArchName.toUpperCase());
+        } catch (final IllegalArgumentException ex) {
+            logger.error("Trying to use LightGBM on an unsupported architecture {}.", cpuArchName, ex);
+            throw ex;
+        }
+    }
+
     /**
      * Loads a single shared library from the Jar.
      *
      * @param sharedLibResourceName library "filename" inside the jar.
+     * @param cpuArchitecture cpu architecture.
      * @throws IOException if any error happens loading the library.
      */
-    static private void loadSharedLibraryFromJar(final String sharedLibResourceName) throws IOException {
+    static private void loadSharedLibraryFromJar(
+            final String sharedLibResourceName,
+            final CpuArchitecture cpuArchitecture
+    ) throws IOException {
 
-        logger.debug("Loading LightGBM shared lib: {}.", sharedLibResourceName);
+        logger.debug("Loading LightGBM shared lib: {} for {}.", sharedLibResourceName, cpuArchitecture);
 
-        final InputStream inputStream = LightGBMUtils.class.getClassLoader().getResourceAsStream(sharedLibResourceName);
+        final InputStream inputStream = LightGBMUtils.class.getClassLoader()
+                .getResourceAsStream(cpuArchitecture.getLgbmNativeLibsFolder() + "/" + sharedLibResourceName);
         final File tempFile = File.createTempFile("lib", ".so");
         final OutputStream outputStream = new FileOutputStream(tempFile);
 

openml-lightgbm/lightgbm-provider/src/main/java/com/feedzai/openml/provider/lightgbm/LightGBMUtils.java

@@ -236,7 +236,7 @@ public void ensureLightgbmScoresAndContributions() throws URISyntaxException, IO
                     .isEqualTo(2);
 
             final double prediction = resultInstance.getValue(9);
-            final Offset<Double> offset = Offset.offset(1.0e-16);
+            final Offset<Double> offset = Offset.offset(1.0e-14);
 
             assertThat(scoreDistribution[1])
                     .as("The score should be expected.")

openml-lightgbm/lightgbm-provider/src/test/java/com/feedzai/openml/provider/lightgbm/LightGBMResultTest.java

@@ -0,0 +1,34 @@
+/*
+ * The copyright of this file belongs to Feedzai. The file cannot be
+ * reproduced in whole or in part, stored in a retrieval system,
+ * transmitted in any form, or by any means electronic, mechanical,
+ * photocopying, or otherwise, without the prior permission of the owner.
+ *
+ * © 2023 Feedzai, Strictly Confidential
+ */
+package com.feedzai.openml.provider.lightgbm;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+/**
+ * Tests the retrieval of cpu architecture.
+ *
+ * @author Artur Pedroso (artur.pedroso@feedzai.com)
+ */
+public class TestCpuArchitecture {
+    @Test
+    public void unknownCpuArchitectureThrowsException() {
+        Assertions.assertThatThrownBy(() -> LightGBMUtils.getCpuArchitecture("x86_64"))
+                .isInstanceOf(IllegalArgumentException.class);
+    }
+
+    @Test
+    public void canReadKnownCpuArchitectures() {
+        Assertions.assertThat(LightGBMUtils.getCpuArchitecture("aarch64"))
+                .isEqualTo(CpuArchitecture.AARCH64);
+
+        Assertions.assertThat(LightGBMUtils.getCpuArchitecture("amd64"))
+                .isEqualTo(CpuArchitecture.AMD64);
+    }
+}