diff --git a/.gitignore b/.gitignore index 171f088a..e8ff1943 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ hs_err_pid* target .DS_Store +dependency-reduced-pom.xml venv diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..c72c9dd6 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,151 @@ +# Mork Benchmarks + +This module contains JMH (Java Microbenchmark Harness) benchmarks for the Mork framework, focusing on performance comparisons between the standard Java `HashSet` and the custom `BitSet` implementation. + +## Benchmark Tests + +### SetAddBenchmark +Compares the performance of adding elements to HashSet vs BitSet across different data sizes (100, 1000, 10000 elements) and fill ratios (50%, 90%). + +**Measures:** Average time per operation in nanoseconds + +### SetContainsBenchmark +Compares the performance of lookup/contains operations between HashSet and BitSet. + +**Measures:** Average time for 1000 lookups in nanoseconds + +### SetIterationBenchmark +Compares the performance of iterating over all elements in HashSet vs BitSet. + +**Measures:** Average time to iterate over all elements in nanoseconds + +### SetRemoveBenchmark +Compares the performance of removing elements from HashSet vs BitSet. + +**Measures:** Average time per operation in nanoseconds + +### SetMemoryBenchmark +Measures memory allocation patterns between HashSet and BitSet across different sizes (1000, 10000, 100000 elements). + +**Measures:** Average and sample time, allocation rate tracking + +### SetMixedOperationsBenchmark +Simulates realistic workloads with mixed operations (add, contains, remove) to compare overall performance. + +**Measures:** Average time per mixed workload in microseconds + +## Building the Benchmarks + +To build the benchmark uber JAR: + +```bash +cd benchmarks +mvn clean package +``` + +This will create `target/benchmarks.jar` containing all benchmarks and dependencies. + +## Running Benchmarks + +### Run All Benchmarks + +```bash +java -jar target/benchmarks.jar +``` + +### Run Specific Benchmark + +```bash +# Run only add benchmarks +java -jar target/benchmarks.jar SetAddBenchmark + +# Run only contains benchmarks +java -jar target/benchmarks.jar SetContainsBenchmark +``` + +### Run with Custom Parameters + +```bash +# Run with specific parameters +java -jar target/benchmarks.jar SetAddBenchmark -p size=1000 -p fillRatio=0.9 + +# Run with specific warmup and measurement iterations +java -jar target/benchmarks.jar -wi 5 -i 10 +``` + +### Run with Profilers + +JMH includes several profilers to get additional insights: + +```bash +# List available profilers +java -jar target/benchmarks.jar -lprof + +# Run with GC profiler +java -jar target/benchmarks.jar -prof gc + +# Run with stack profiler +java -jar target/benchmarks.jar -prof stack + +# Run with allocation profiler (requires -javaagent) +java -jar target/benchmarks.jar -prof "async:libPath=/path/to/libasyncProfiler.so;output=flamegraph" +``` + +### Save Results + +```bash +# Save results to JSON +java -jar target/benchmarks.jar -rf json -rff results.json + +# Save results to CSV +java -jar target/benchmarks.jar -rf csv -rff results.csv +``` + +## Understanding Results + +The benchmarks measure: + +- **CPU Performance**: All benchmarks measure execution time (throughput) +- **Memory Impact**: SetMemoryBenchmark runs with specific GC settings to track allocation patterns +- **Different Use Cases**: + - Small datasets (100-1000 elements): Typical for small optimization problems + - Medium datasets (10000 elements): Common in many algorithms + - Large datasets (100000 elements): Stress testing for memory benchmarks + +### Expected Results + +**BitSet Advantages:** +- Lower memory footprint for dense sets (high fill ratios) +- Faster operations when elements are small integers +- More predictable performance characteristics + +**HashSet Advantages:** +- Better for sparse sets (low fill ratios) +- Can handle any object type, not just integers +- No capacity limit requirement + +## JMH Options + +Common JMH command-line options: + +- `-h`: Display help +- `-l`: List available benchmarks +- `-lprof`: List available profilers +- `-wi `: Number of warmup iterations +- `-i `: Number of measurement iterations +- `-f `: Number of forks +- `-t `: Number of threads +- `-p =`: Set benchmark parameter + +## Requirements + +- Java 21 or higher +- Maven 3.6+ +- At least 2GB of available memory for running benchmarks + +## Notes + +- Benchmarks may take several minutes to complete +- Results can vary based on JVM version, hardware, and system load +- For production decisions, always run benchmarks on target hardware +- The benchmarks use a fixed random seed (42) for reproducibility diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml new file mode 100644 index 00000000..61746859 --- /dev/null +++ b/benchmarks/pom.xml @@ -0,0 +1,92 @@ + + + 4.0.0 + + + mork-benchmarks + 0.22-SNAPSHOT + mork-benchmarks + Benchmarking tests for Mork framework using JMH + + + + es.urjc.etsii.grafo + mork-parent + 0.22-SNAPSHOT + + + + 1.37 + benchmarks + 21 + 21 + + + + + + es.urjc.etsii.grafo + mork-common + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetAddBenchmark.java b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetAddBenchmark.java new file mode 100644 index 00000000..fcb7c8aa --- /dev/null +++ b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetAddBenchmark.java @@ -0,0 +1,62 @@ +package es.urjc.etsii.grafo.benchmarks; + +import es.urjc.etsii.grafo.util.collections.BitSet; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.HashSet; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * Benchmark comparing add operation performance between HashSet and BitSet + * for different data sizes. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 3) +@Fork(1) +public class SetAddBenchmark { + + @Param({"100", "1000", "10000"}) + private int size; + + @Param({"0.5", "0.9"}) // Fill ratio + private double fillRatio; + + private int[] elements; + private int capacity; + + @Setup(Level.Trial) + public void setup() { + capacity = size; + int numElements = (int) (size * fillRatio); + elements = new int[numElements]; + Random random = new Random(42); + + // Generate random unique elements + for (int i = 0; i < numElements; i++) { + elements[i] = random.nextInt(capacity); + } + } + + @Benchmark + public void hashSetAdd(Blackhole blackhole) { + HashSet set = new HashSet<>(capacity); + for (int element : elements) { + set.add(element); + } + blackhole.consume(set); + } + + @Benchmark + public void bitSetAdd(Blackhole blackhole) { + BitSet set = new BitSet(capacity); + for (int element : elements) { + set.add(element); + } + blackhole.consume(set); + } +} diff --git a/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetContainsBenchmark.java b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetContainsBenchmark.java new file mode 100644 index 00000000..ffa6ac80 --- /dev/null +++ b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetContainsBenchmark.java @@ -0,0 +1,77 @@ +package es.urjc.etsii.grafo.benchmarks; + +import es.urjc.etsii.grafo.util.collections.BitSet; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.HashSet; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * Benchmark comparing contains/lookup operation performance between HashSet and BitSet. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 3) +@Fork(1) +public class SetContainsBenchmark { + + @Param({"100", "1000", "10000"}) + private int size; + + @Param({"0.5", "0.9"}) // Fill ratio + private double fillRatio; + + private HashSet hashSet; + private BitSet bitSet; + private int[] lookupElements; + private int capacity; + + @Setup(Level.Trial) + public void setup() { + capacity = size; + int numElements = (int) (size * fillRatio); + Random random = new Random(42); + + // Create and populate sets + hashSet = new HashSet<>(capacity); + bitSet = new BitSet(capacity); + + for (int i = 0; i < numElements; i++) { + int element = random.nextInt(capacity); + hashSet.add(element); + bitSet.add(element); + } + + // Create lookup elements (mix of existing and non-existing) + lookupElements = new int[1000]; + for (int i = 0; i < lookupElements.length; i++) { + lookupElements[i] = random.nextInt(capacity); + } + } + + @Benchmark + public void hashSetContains(Blackhole blackhole) { + int count = 0; + for (int element : lookupElements) { + if (hashSet.contains(element)) { + count++; + } + } + blackhole.consume(count); + } + + @Benchmark + public void bitSetContains(Blackhole blackhole) { + int count = 0; + for (int element : lookupElements) { + if (bitSet.contains(element)) { + count++; + } + } + blackhole.consume(count); + } +} diff --git a/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetIterationBenchmark.java b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetIterationBenchmark.java new file mode 100644 index 00000000..5481119c --- /dev/null +++ b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetIterationBenchmark.java @@ -0,0 +1,65 @@ +package es.urjc.etsii.grafo.benchmarks; + +import es.urjc.etsii.grafo.util.collections.BitSet; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.HashSet; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * Benchmark comparing iteration performance between HashSet and BitSet. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 3) +@Fork(1) +public class SetIterationBenchmark { + + @Param({"100", "1000", "10000"}) + private int size; + + @Param({"0.5", "0.9"}) // Fill ratio + private double fillRatio; + + private HashSet hashSet; + private BitSet bitSet; + + @Setup(Level.Trial) + public void setup() { + int capacity = size; + int numElements = (int) (size * fillRatio); + Random random = new Random(42); + + // Create and populate sets + hashSet = new HashSet<>(capacity); + bitSet = new BitSet(capacity); + + for (int i = 0; i < numElements; i++) { + int element = random.nextInt(capacity); + hashSet.add(element); + bitSet.add(element); + } + } + + @Benchmark + public void hashSetIteration(Blackhole blackhole) { + int sum = 0; + for (Integer element : hashSet) { + sum += element; + } + blackhole.consume(sum); + } + + @Benchmark + public void bitSetIteration(Blackhole blackhole) { + int sum = 0; + for (Integer element : bitSet) { + sum += element; + } + blackhole.consume(sum); + } +} diff --git a/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetMemoryBenchmark.java b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetMemoryBenchmark.java new file mode 100644 index 00000000..c4440141 --- /dev/null +++ b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetMemoryBenchmark.java @@ -0,0 +1,70 @@ +package es.urjc.etsii.grafo.benchmarks; + +import es.urjc.etsii.grafo.util.collections.BitSet; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.HashSet; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * Benchmark measuring memory allocation between HashSet and BitSet. + * This measures the memory impact through allocation rate tracking. + */ +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Thread) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 3) +@Fork(value = 1, jvmArgsAppend = {"-XX:+UseG1GC", "-Xms2g", "-Xmx2g"}) +public class SetMemoryBenchmark { + + @Param({"1000", "10000", "100000"}) + private int size; + + @Param({"0.5", "0.9"}) // Fill ratio + private double fillRatio; + + private int[] elements; + private int capacity; + + @Setup(Level.Invocation) + public void setup() { + capacity = size; + int numElements = (int) (size * fillRatio); + elements = new int[numElements]; + Random random = new Random(42); + + // Generate random unique elements + for (int i = 0; i < numElements; i++) { + elements[i] = random.nextInt(capacity); + } + } + + @Benchmark + public void hashSetMemory(Blackhole blackhole) { + HashSet set = new HashSet<>(capacity); + for (int element : elements) { + set.add(element); + } + // Perform operations to ensure memory is actually allocated + for (int element : elements) { + set.contains(element); + } + blackhole.consume(set); + } + + @Benchmark + public void bitSetMemory(Blackhole blackhole) { + BitSet set = new BitSet(capacity); + for (int element : elements) { + set.add(element); + } + // Perform operations to ensure memory is actually allocated + for (int element : elements) { + set.contains(element); + } + blackhole.consume(set); + } +} diff --git a/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetMixedOperationsBenchmark.java b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetMixedOperationsBenchmark.java new file mode 100644 index 00000000..41832fc4 --- /dev/null +++ b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetMixedOperationsBenchmark.java @@ -0,0 +1,105 @@ +package es.urjc.etsii.grafo.benchmarks; + +import es.urjc.etsii.grafo.util.collections.BitSet; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.HashSet; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * Benchmark comparing mixed operations (add, contains, remove) between HashSet and BitSet. + * This simulates realistic workloads where multiple operations are performed. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 3) +@Fork(1) +public class SetMixedOperationsBenchmark { + + @Param({"1000", "10000"}) + private int size; + + private int[] addElements; + private int[] lookupElements; + private int[] removeElements; + private int capacity; + + @Setup(Level.Trial) + public void setup() { + capacity = size; + Random random = new Random(42); + + // Generate elements for different operations + addElements = new int[size]; + lookupElements = new int[size / 2]; + removeElements = new int[size / 4]; + + for (int i = 0; i < size; i++) { + addElements[i] = random.nextInt(capacity); + } + + for (int i = 0; i < lookupElements.length; i++) { + lookupElements[i] = random.nextInt(capacity); + } + + for (int i = 0; i < removeElements.length; i++) { + removeElements[i] = addElements[i]; + } + } + + @Benchmark + public void hashSetMixedOps(Blackhole blackhole) { + HashSet set = new HashSet<>(capacity); + + // Add phase + for (int element : addElements) { + set.add(element); + } + + // Lookup phase + int found = 0; + for (int element : lookupElements) { + if (set.contains(element)) { + found++; + } + } + + // Remove phase + for (int element : removeElements) { + set.remove(element); + } + + blackhole.consume(set); + blackhole.consume(found); + } + + @Benchmark + public void bitSetMixedOps(Blackhole blackhole) { + BitSet set = new BitSet(capacity); + + // Add phase + for (int element : addElements) { + set.add(element); + } + + // Lookup phase + int found = 0; + for (int element : lookupElements) { + if (set.contains(element)) { + found++; + } + } + + // Remove phase + for (int element : removeElements) { + set.remove(element); + } + + blackhole.consume(set); + blackhole.consume(found); + } +} diff --git a/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetRemoveBenchmark.java b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetRemoveBenchmark.java new file mode 100644 index 00000000..0426f905 --- /dev/null +++ b/benchmarks/src/main/java/es/urjc/etsii/grafo/benchmarks/SetRemoveBenchmark.java @@ -0,0 +1,74 @@ +package es.urjc.etsii.grafo.benchmarks; + +import es.urjc.etsii.grafo.util.collections.BitSet; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.HashSet; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * Benchmark comparing remove operation performance between HashSet and BitSet. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 3) +@Fork(1) +public class SetRemoveBenchmark { + + @Param({"100", "1000", "10000"}) + private int size; + + @Param({"0.5", "0.9"}) // Fill ratio + private double fillRatio; + + private int[] elementsToRemove; + private int capacity; + private int[] initialElements; + + @Setup(Level.Trial) + public void setup() { + capacity = size; + int numElements = (int) (size * fillRatio); + Random random = new Random(42); + + // Generate initial elements + initialElements = new int[numElements]; + for (int i = 0; i < numElements; i++) { + initialElements[i] = random.nextInt(capacity); + } + + // Generate elements to remove (half of the initial elements) + elementsToRemove = new int[numElements / 2]; + for (int i = 0; i < elementsToRemove.length; i++) { + elementsToRemove[i] = initialElements[i]; + } + } + + @Benchmark + public void hashSetRemove(Blackhole blackhole) { + HashSet set = new HashSet<>(capacity); + for (int element : initialElements) { + set.add(element); + } + for (int element : elementsToRemove) { + set.remove(element); + } + blackhole.consume(set); + } + + @Benchmark + public void bitSetRemove(Blackhole blackhole) { + BitSet set = new BitSet(capacity); + for (int element : initialElements) { + set.add(element); + } + for (int element : elementsToRemove) { + set.remove(element); + } + blackhole.consume(set); + } +} diff --git a/pom.xml b/pom.xml index 14b1d197..76ca1b22 100644 --- a/pom.xml +++ b/pom.xml @@ -23,6 +23,7 @@ autoconfig integration-tests aggregate-report + benchmarks