Pending latest changes from Widehopf: Dev

.github/workflows/docker.yaml

@@ -2,15 +2,16 @@ name: docker build
 
 on:
   schedule:
-  - cron: "0 10 * * *"
+    - cron: "0 5 * * 1"
   push:
     branches:
-    - "**"
+      - "dev"
+      - "sid"
     tags:
-    - "v*.*.*"
+      - "v*.*.*"
   pull_request:
     branches:
-    - "dev"
+      - "dev"
 
 jobs:
   docker:
@@ -21,8 +22,8 @@ jobs:
     - name: Set variables useful for later
       id: useful_vars
       run: |-
-        echo "::set-output name=timestamp::$(date +%s)"
-        echo "::set-output name=short_sha::${GITHUB_SHA::8}"
+        echo "name=timestamp::$(date +%s)" >> $GITHUB_OUTPUT
+        echo "name=short_sha::${GITHUB_SHA::8}" >> $GITHUB_OUTPUT
     - name: Checkout
       uses: actions/checkout@v3
     - name: Docker meta
@@ -52,13 +53,6 @@ jobs:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Cache Docker layers
-      uses: actions/cache@v3
-      with:
-        path: /tmp/.buildx-cache
-        key: ${{ runner.os }}-buildx-${{ github.sha }}
-        restore-keys: |
-          ${{ runner.os }}-buildx-
     - name: Build and push
       uses: docker/build-push-action@v4
       with:
@@ -67,5 +61,6 @@ jobs:
         tags: ${{ steps.docker_meta.outputs.tags }}
         labels: ${{ steps.docker_meta.outputs.labels }}
         platforms: linux/amd64,linux/arm64
-        cache-from: type=local,src=/tmp/.buildx-cache
-        cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
+        build-args: |
+          BUILDKIT_CONTEXT_KEEP_GIT_DIR=true
+        outputs: type=image,oci-mediatypes=true,compression=zstd,compression-level=3,force-compression=true

Dockerfile

@@ -1,30 +1,21 @@
-FROM debian:bookworm-20240311 AS builder
-RUN apt-get update && \
-    apt-get install -y git wget pkg-config autoconf gcc make libusb-1.0-0-dev librtlsdr-dev librtlsdr0 libncurses-dev zlib1g-dev zlib1g libzstd-dev libzstd1
+FROM ghcr.io/wiedehopf/readsb-builder:latest AS builder
 
-# install jemalloc
-RUN JEMALLOC_BDIR=$(mktemp -d) && \
-    git clone --depth 1 https://github.com/jemalloc/jemalloc $JEMALLOC_BDIR && \
-    cd $JEMALLOC_BDIR && \
-    ./autogen.sh && \
-    ./configure --with-lg-page=14 && \
-    make -j$(nproc) && \
-    make install && \
-    rm -rf $JEMALLOC_BDIR
-
-# install readsb
-RUN mkdir -p /app/git
-COPY . /app/git
-RUN cd /app/git && \
+SHELL ["/bin/bash", "-x", "-o", "pipefail", "-c"]
+RUN --mount=type=bind,source=.,target=/app/git \
+    cd /app/git && \
     READSB_BUILD_DIR=$(mktemp -d) && \
-    cp -r /app/git/* $READSB_BUILD_DIR && \
+    cp -aT /app/git $READSB_BUILD_DIR && \
     cd $READSB_BUILD_DIR && \
-    make -j$(nproc) RTLSDR=yes OPTIMIZE="-O2" && \
+    [[ $(uname -m) == x86_64 ]] && MARCH=" -march=nehalem" || MARCH="" && \
+    make -j$(nproc) RTLSDR=yes OPTIMIZE="-O2 $MARCH" && \
     mv readsb /usr/local/bin && \
     mv viewadsb /usr/local/bin && \
     chmod +x /usr/local/bin/viewadsb /usr/local/bin/readsb && \
     make clean && \
-    make -j$(nproc) PRINT_UUIDS=yes TRACKS_UUID=yes OPTIMIZE="-O2" && \
+    make -j$(nproc) WITH_UUIDS=yes OPTIMIZE="-O2 $MARCH" && \
+    mv readsb /usr/local/bin/readsb-with-uuids && \
+    make clean && \
+    make -j$(nproc) PRINT_UUIDS=yes TRACKS_UUID=yes OPTIMIZE="-O2 $MARCH" && \
     mv readsb /usr/local/bin/readsb-uuid && \
     mv viewadsb /usr/local/bin/viewadsb-uuid && \
     chmod +x /usr/local/bin/viewadsb-uuid && \
@@ -35,18 +26,22 @@ RUN cd /app/git && \
     true
 
 FROM debian:bookworm-slim
-RUN apt-get update && \
+RUN \
+    --mount=type=bind,from=builder,source=/,target=/builder/ \
+    apt-get update && \
     apt-get -y install --no-install-recommends \
-    wget \
     librtlsdr0 libncurses6 zlib1g libzstd1 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    mkdir -p /run/readsb
+    mkdir -p /run/readsb && \
+    cp /builder/usr/local/bin/readsb* /usr/local/bin/ && \
+    cp /builder/usr/local/bin/viewadsb* /usr/local/bin/ && \
+    mkdir -p  /usr/local/share/tar1090 && \
+    cp /builder/usr/local/share/tar1090/aircraft.csv.gz /usr/local/share/tar1090/aircraft.csv.gz && \
+    cp /builder/usr/local/lib/libjemalloc.so.2 /usr/local/lib/libjemalloc.so.2 && \
+    true
 
 ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so.2
-COPY --from=builder /usr/local/bin/readsb* /usr/local/bin/
-COPY --from=builder /usr/local/bin/viewadsb* /usr/local/bin/
-COPY --from=builder /usr/local/share/tar1090/aircraft.csv.gz /usr/local/share/tar1090/aircraft.csv.gz
-COPY --from=builder /usr/local/lib/libjemalloc.so.2 /usr/local/lib/libjemalloc.so.2
+ENV MALLOC_CONF=narenas:1,tcache:false
 
 ENTRYPOINT ["/usr/local/bin/readsb"]

Makefile

@@ -8,114 +8,155 @@ PLUTOSDR ?= no
 SOAPYSDR ?= no
 AGGRESSIVE ?= no
 HAVE_BIASTEE ?= no
+# TRACKS_UUID saves the uuid for each position to historic / current traces (often multiple uuids will send the
+# same position, in this case the first uuid to contribute the position will be noted as only one
+# uuid is saved)
+# this will invalidate your persistent state so changing this option will mean you lose up to 24h of historic
+# traces
 TRACKS_UUID ?= no
+# WITH_UUIDS saves the last up to RECENT_RECEICER_IDS (default 32) with each aircraft
+# this increase memory usage a bit but allows filtering the API by uuid (&filter_uuid)
+WITH_UUIDS ?= no
+# PRINT_UUIDS implies WITH_UUIDS and also prints all uuids an aircraft is heard from into
+# aircraft.json and other json output
 PRINT_UUIDS ?= no
 
 DIALECT = -std=c11
-CFLAGS = $(DIALECT) -W -D_GNU_SOURCE -D_DEFAULT_SOURCE -Wall -Werror -fno-common -O2
+CFLAGS = $(DIALECT) -W -Wall -Werror -fno-common -O2
 CFLAGS += -DMODES_READSB_VERSION=\"$(READSB_VERSION)\"
-CFLAGS += -Wdate-time -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-strong -Wformat -Werror=format-security
-
-LIBS = -pthread -lpthread -lm -lrt -lzstd
+CFLAGS += -DREADSB_SHORT_VERSION=\"$(shell cat version)\"
+CFLAGS += -DREADSB_SHORT_COMMIT=\"$(shell git describe --abbrev --dirty --always || echo nogit)\"
+CFLAGS += -Wdate-time -fstack-protector-strong -Wformat -Werror=format-security
+
+# Platform-specific settings
+UNAME := $(shell uname)
+ifeq ($(UNAME), Linux)
+    CFLAGS += -D_GNU_SOURCE -D_DEFAULT_SOURCE
+    CFLAGS += -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2
+    LIBS = -pthread -lpthread -lm -lrt -lzstd
+else ifeq ($(UNAME), Darwin)
+    CFLAGS += -D_DARWIN_C_SOURCE
+    CFLAGS += -I./compat/apple  # For epoll_shim.h
+    CFLAGS += -I/opt/homebrew/include  # For zstd.h
+    LDFLAGS += -L/opt/homebrew/lib
+    LIBS = -pthread -lpthread -lm -lzstd
+    COMPAT += compat/apple/epoll_shim.o
+else
+    LIBS = -pthread -lpthread -lm -lrt -lzstd
+endif
 
 ifeq ($(ZLIB_STATIC), yes)
 	LIBS += -l:libz.a
 else
-	LIBS += -lz
+    LIBS += -lz
 endif
 
 ifeq ($(shell $(CC) -c feature_test.c -o feature_test.o -Wno-format-truncation -Werror >/dev/null 2>&1 && echo 1 || echo 0), 1)
-	CFLAGS += -Wno-format-truncation
+    CFLAGS += -Wno-format-truncation
 endif
 
 ifeq ($(shell uname -m | grep -qs -e arm -e aarch64 >/dev/null 2>&1 && echo 1 || echo 0), 1)
-  CFLAGS += -DSC16Q11_TABLE_BITS=8
+    CFLAGS += -DSC16Q11_TABLE_BITS=8
 endif
 
 ifeq ($(DISABLE_INTERACTIVE), yes)
-  CFLAGS += -DDISABLE_INTERACTIVE
+    CFLAGS += -DDISABLE_INTERACTIVE
 else
-  LIBS += $(shell pkg-config --libs ncurses)
+    ifeq ($(UNAME), Darwin)
+        LIBS += $(shell pkg-config --libs ncurses 2>/dev/null || echo -lncurses)
+    else
+        LIBS += $(shell pkg-config --libs ncurses)
+    endif
 endif
 
 # only disable workaround if zerocopy is disabled in librtlsdr, otherwise expect significantly increased CPU use
 ifeq ($(DISABLE_RTLSDR_ZEROCOPY_WORKAROUND), yes)
-  CFLAGS += -DDISABLE_RTLSDR_ZEROCOPY_WORKAROUND
+    CFLAGS += -DDISABLE_RTLSDR_ZEROCOPY_WORKAROUND
 endif
 
-ifeq ($(HISTORY), yes)
-  CFLAGS += -DALL_JSON=1
+ifeq ($(LTO), yes)
+	CFLAGS += -flto
+	LDFLAGS += -flto
 endif
 
-ifneq ($(PREAMBLE_THRESHOLD_DEFAULT),)
-  CFLAGS += -DPREAMBLE_THRESHOLD_DEFAULT=$(PREAMBLE_THRESHOLD_DEFAULT)
+ifeq ($(HISTORY), yes)
+    CFLAGS += -DALL_JSON=1
 endif
 
-ifneq ($(GLOBE_PERM_IVAL),)
-  CFLAGS += -DGLOBE_PERM_IVAL=$(GLOBE_PERM_IVAL)
+ifneq ($(PREAMBLE_THRESHOLD_DEFAULT),)
+    CFLAGS += -DPREAMBLE_THRESHOLD_DEFAULT=$(PREAMBLE_THRESHOLD_DEFAULT)
 endif
 
 ifneq ($(TRACE_THREADS),)
-  CFLAGS += -DTRACE_THREADS=$(TRACE_THREADS)
+    CFLAGS += -DTRACE_THREADS=$(TRACE_THREADS)
 endif
 
 ifneq ($(TRACE_RECENT_POINTS),)
-  CFLAGS += -DTRACE_RECENT_POINTS=$(TRACE_RECENT_POINTS)
+    CFLAGS += -DTRACE_RECENT_POINTS=$(TRACE_RECENT_POINTS)
 endif
 
 ifneq ($(AIRCRAFT_HASH_BITS),)
-  CFLAGS += -DAIRCRAFT_HASH_BITS=$(AIRCRAFT_HASH_BITS)
+    CFLAGS += -DAIRCRAFT_HASH_BITS=$(AIRCRAFT_HASH_BITS)
 endif
 
 ifeq ($(STATS_PHASE),yes)
-  CFLAGS += -DSTATS_PHASE
+    CFLAGS += -DSTATS_PHASE
 endif
 
 ifeq ($(TRACKS_UUID), yes)
-  CFLAGS += -DTRACKS_UUID
+    CFLAGS += -DTRACKS_UUID
+endif
+
+ifeq ($(WITH_UUIDS), yes)
+    CFLAGS += -DWITH_UUIDS
+endif
+
+ifneq ($(RECENT_RECEIVER_IDS),)
+    CFLAGS += -DRECENT_RECEIVER_IDS=$(RECENT_RECEIVER_IDS)
 endif
 
 ifeq ($(PRINT_UUIDS), yes)
-  CFLAGS += -DPRINT_UUIDS
+    CFLAGS += -DPRINT_UUIDS
 endif
 
 ifneq ($(RECENT_RECEIVER_IDS),)
-  CFLAGS += -DRECENT_RECEIVER_IDS=$(RECENT_RECEIVER_IDS)
+    CFLAGS += -DRECENT_RECEIVER_IDS=$(RECENT_RECEIVER_IDS)
 endif
 
 ifeq ($(RTLSDR), yes)
-  SDR_OBJ += sdr_rtlsdr.o
-  CFLAGS += -DENABLE_RTLSDR
+    SDR_OBJ += sdr_rtlsdr.o
+    CFLAGS += -DENABLE_RTLSDR
 
-  ifeq ($(HAVE_BIASTEE), yes)
-    CFLAGS += -DENABLE_RTLSDR_BIASTEE
-  endif
+    ifeq ($(HAVE_BIASTEE), yes)
+        CFLAGS += -DENABLE_RTLSDR_BIASTEE
+    endif
 
-  ifdef RTLSDR_PREFIX
-    CFLAGS += -I$(RTLSDR_PREFIX)/include
-    LDFLAGS += -L$(RTLSDR_PREFIX)/lib
-  else
-    CFLAGS += $(shell pkg-config --cflags librtlsdr)
-    LDFLAGS += $(shell pkg-config --libs-only-L librtlsdr)
-  endif
+    ifdef RTLSDR_PREFIX
+        CFLAGS += -I$(RTLSDR_PREFIX)/include
+        LDFLAGS += -L$(RTLSDR_PREFIX)/lib
+    else
+        CFLAGS += $(shell pkg-config --cflags librtlsdr)
+        LDFLAGS += $(shell pkg-config --libs-only-L librtlsdr)
+    endif
 
-  ifeq ($(STATIC), yes)
-    LIBS_SDR += -Wl,-Bstatic -lrtlsdr -Wl,-Bdynamic -lusb-1.0
-  else
-    LIBS_SDR += -lrtlsdr -lusb-1.0
-  endif
+	# static linking not well supported, use at own risk
+    ifeq ($(STATIC), yes)
+		LIBS_SDR += -Wl,-Bstatic -lrtlsdr -Wl,-Bdynamic -lusb-1.0
+    else
+        LIBS_SDR += -lrtlsdr -lusb-1.0
+    endif
 endif
 
 ifeq ($(BLADERF), yes)
-  SDR_OBJ += sdr_bladerf.o sdr_ubladerf.o
-  CFLAGS += $(shell pkg-config --cflags libbladeRF) -DENABLE_BLADERF
-  LIBS_SDR += $(shell pkg-config --libs libbladeRF)
+    SDR_OBJ += sdr_bladerf.o sdr_ubladerf.o
+    CFLAGS += $(shell pkg-config --cflags libbladeRF) -DENABLE_BLADERF
+    LIBS_SDR += $(shell pkg-config --libs libbladeRF)
 endif
 
 ifeq ($(HACKRF), yes)
-  SDR_OBJ += sdr_hackrf.o
-  CFLAGS += $(shell pkg-config --cflags libhackrf) -DENABLE_HACKRF
-  LIBS_SDR += $(shell pkg-config --libs libhackrf)
+    SDR_OBJ += sdr_hackrf.o
+    CFLAGS += $(shell pkg-config --cflags libhackrf) -DENABLE_HACKRF
+    LIBS_SDR += $(shell pkg-config --libs libhackrf)
 endif
 
 ifeq ($(PLUTOSDR), yes)
@@ -125,9 +166,9 @@ ifeq ($(PLUTOSDR), yes)
 endif
 
 ifeq ($(SOAPYSDR), yes)
-  SDR_OBJ += sdr_soapy.o
-  CFLAGS += $(shell pkg-config --cflags SoapySDR) -DENABLE_SOAPYSDR
-  LIBS_SDR += $(shell pkg-config --libs SoapySDR)
+    SDR_OBJ += sdr_soapy.o
+    CFLAGS += $(shell pkg-config --cflags SoapySDR) -DENABLE_SOAPYSDR
+    LIBS_SDR += $(shell pkg-config --libs SoapySDR)
 endif
 
 # add custom overrides if user defines them
@@ -147,13 +188,10 @@ readsb.o: readsb.c *.h .version
 %.o: %.c *.h
 	$(CC) $(CFLAGS) -c $< -o $@
 
-minilzo.o: minilzo/minilzo.c minilzo/minilzo.h
-	$(CC) $(CFLAGS) -c $< -o $@
-
 readsb: readsb.o argp.o anet.o interactive.o mode_ac.o mode_s.o comm_b.o json_out.o net_io.o crc.o demod_2400.o \
 	uat2esnt/uat2esnt.o uat2esnt/uat_decode.o \
 	stats.o cpr.o icao_filter.o track.o util.o fasthash.o convert.o sdr_ifile.o sdr_beast.o sdr.o ais_charset.o \
-	globe_index.o geomag.o receiver.o aircraft.o api.o minilzo.o threadpool.o \
+	globe_index.o geomag.o receiver.o aircraft.o api.o threadpool.o \
 	$(SDR_OBJ) $(COMPAT)
 	$(CC) -o $@ $^ $(LDFLAGS) $(LIBS) $(LIBS_SDR) $(OPTIMIZE)
 
@@ -162,18 +200,23 @@ viewadsb: readsb
 	cp readsb viewadsb
 
 clean:
-	rm -f *.o uat2esnt/*.o compat/clock_gettime/*.o compat/clock_nanosleep/*.o readsb viewadsb cprtests crctests convert_benchmark
+	rm -f *.o uat2esnt/*.o compat/clock_gettime/*.o compat/clock_nanosleep/*.o compat/apple/*.o readsb viewadsb cprtests crctests convert_benchmark
+
+test: cprtest crctest
 
 cprtest: cprtests
 	./cprtests
 
+crctest: cprtests
+	./cprtests
+
 cprtests: cpr.o cprtests.o
 	$(CC) $(CFLAGS) -o $@ $^ -lm
 
 crctests: crc.c crc.h
 	$(CC) $(CFLAGS) -DCRCDEBUG -o $@ $<
 
-benchmarks: convert_benchmark
+benchmarks: oneoff/convert_benchmark
 	./convert_benchmark
 
 oneoff/convert_benchmark: oneoff/convert_benchmark.o convert.o util.o