From 84a01688703a159a18462b9efd81e06ace9354ac Mon Sep 17 00:00:00 2001 From: Adriano dos Santos Fernandes Date: Tue, 6 Sep 2022 08:50:22 -0300 Subject: [PATCH] Split Android package into two phases, second one to run in emulator. Add GitHub action to run second phase in MacOS machine, where Android emulator can run. Currently this is done for ARM 32/64 builds. --- .github/workflows/main.yml | 175 ++++++++++++++++++ .../arch-specific/android/AfterUntar.sh | 6 +- .../android/BuildFinalPackage.sh | 68 +++++++ .../arch-specific/android/BuildPackage.sh | 46 ++--- .../arch-specific/android/CreateEmulator.sh | 4 + .../arch-specific/android/DeleteEmulator.sh | 4 + .../arch-specific/android/StartEmulator.sh | 6 + builds/posix/make.android.arm64 | 7 +- builds/posix/make.android.arme | 8 +- 9 files changed, 281 insertions(+), 43 deletions(-) create mode 100755 builds/install/arch-specific/android/BuildFinalPackage.sh create mode 100755 builds/install/arch-specific/android/CreateEmulator.sh create mode 100755 builds/install/arch-specific/android/DeleteEmulator.sh create mode 100755 builds/install/arch-specific/android/StartEmulator.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 854a6c4a40..4ecacb53a6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -547,3 +547,178 @@ jobs: token: ${{ secrets.SNAPSHOT_RELEASE_TOKEN }} files: | gen/artifacts/Firebird-*-macos-x64*.pkg + + build-android-initial: + name: build-android-${{ matrix.arch }}-initial + runs-on: ubuntu-22.04 + env: + NDK: /home/runner/Android/Ndk + ARCH_BITS: ${{ (matrix.arch == 'arm32' && 32) || 64 }} + FB_PREFIX: ${{ (matrix.arch == 'arm32' && 'arme') || 'arm64' }} + + strategy: + fail-fast: false + matrix: + arch: + - arm32 + - arm64 + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 10 + + - name: Prepare - install Linux tools + run: | + sudo apt-get install libtool-bin libtomcrypt1 libtomcrypt-dev libtommath1 libtommath-dev libicu-dev zlib1g-dev + + - name: Prepare - Download and install Android NDK + run: | + mkdir downloads + pushd downloads + curl -OL https://dl.google.com/android/repository/android-ndk-r25b-linux.zip + mkdir -p $NDK + unzip android-ndk-r25b-linux.zip -d $NDK && f=("$NDK"/*) && mv "$NDK"/*/* "$NDK" && rmdir "${f[@]}" + + - name: Build + run: | + ./autogen.sh --prefix=/opt/firebird --enable-binreloc --with-cross-build=android.$FB_PREFIX --without-editline + make -j4 + make CROSS_OUT=Y tests -j4 + builds/install/arch-specific/android/BuildPackage.sh $ARCH_BITS + + - name: Upload initial build + uses: actions/upload-artifact@v3 + with: + name: firebird-android-initial-${{ matrix.arch }} + path: | + gen/Firebird-*-android-initial-${{ matrix.arch }}*.tar.gz + gen/Make.Version + + build-android-final: + name: build-android-${{ matrix.arch }}-final + needs: build-android-initial + runs-on: macos-latest + env: + ARCH_BITS: ${{ (matrix.arch == 'arm32' && 32) || 64 }} + + strategy: + fail-fast: false + matrix: + arch: + - arm32 + - arm64 + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 10 + + - name: Download initial build + uses: actions/download-artifact@v3 + with: + name: firebird-android-initial-${{ matrix.arch }} + path: gen + + - name: Cache - AVD + uses: actions/cache@v3 + id: avd-cache + with: + path: | + ~/.android/avd/* + ~/.android/adb* + key: avd-v3 + + - name: Create AVD and generate snapshot for caching + if: steps.avd-cache.outputs.cache-hit != 'true' + uses: reactivecircus/android-emulator-runner@v2 + with: + avd-name: firebird-builder + arch: x86_64 + api-level: 30 + target: google_apis + ndk: 25.1.8937393 + emulator-options: -no-window -noaudio -no-boot-anim -camera-back none -port 5554 + script: | + echo "Generated AVD snapshot for caching." + + - name: Create necessary files in Android emulator + uses: reactivecircus/android-emulator-runner@v2 + with: + avd-name: firebird-builder + arch: x86_64 + api-level: 30 + target: google_apis + ndk: 25.1.8937393 + emulator-options: -no-snapshot-save -no-window -noaudio -no-boot-anim -camera-back none -port 5554 + script: | + adb root + NDK=/Users/runner/Library/Android/sdk/ndk/25.1.8937393 builds/install/arch-specific/android/BuildFinalPackage.sh $ARCH_BITS + + - name: Upload installer + uses: actions/upload-artifact@v3 + with: + name: firebird-android-${{ matrix.arch }} + path: gen/Firebird-*-android-${{ matrix.arch }}.tar.gz + + - name: Upload installer (withDebugSymbols) + uses: actions/upload-artifact@v3 + with: + name: firebird-android-${{ matrix.arch }}-withDebugSymbols + path: gen/Firebird-*-android-${{ matrix.arch }}-withDebugSymbols.tar.gz + + - name: Delete initial build + uses: geekyeggo/delete-artifact@v1 + with: + name: firebird-android-initial-${{ matrix.arch }} + + - name: Snapshot - prepare + id: snapshot_prepare + if: | + github.repository == 'FirebirdSQL/firebird' && + github.event.head_commit.message == 'increment build number' && + github.ref_name == 'master' + run: | + echo "::set-output name=snapshot_name::${{ github.ref_name }}" + + - name: Snapshot - delete old assets + uses: asfernandes/delete-release-assets@v2 + if: steps.snapshot_prepare.outputs.snapshot_name + with: + repository: FirebirdSQL/snapshots + token: ${{ secrets.SNAPSHOT_RELEASE_TOKEN }} + tag: snapshot-${{ steps.snapshot_prepare.outputs.snapshot_name }} + fail-if-no-release: false + fail-if-no-assets: false + assets: | + Firebird-*-android-${{ matrix.arch }}*.tar.gz + + - name: Snapshot - release + id: snapshot_release_try1 + uses: softprops/action-gh-release@v1 + if: steps.snapshot_prepare.outputs.snapshot_name + continue-on-error: true + with: + repository: FirebirdSQL/snapshots + name: Latest snapshots for ${{ steps.snapshot_prepare.outputs.snapshot_name }} + tag_name: snapshot-${{ steps.snapshot_prepare.outputs.snapshot_name }} + prerelease: true + token: ${{ secrets.SNAPSHOT_RELEASE_TOKEN }} + files: | + gen/Firebird-*-android-${{ matrix.arch }}*.tar.gz + + - name: Snapshot - release (retry) + uses: softprops/action-gh-release@v1 + if: | + steps.snapshot_prepare.outputs.snapshot_name && + steps.snapshot_release_try1.outcome == 'failure' + with: + repository: FirebirdSQL/snapshots + name: Latest snapshots for ${{ steps.snapshot_prepare.outputs.snapshot_name }} + tag_name: snapshot-${{ steps.snapshot_prepare.outputs.snapshot_name }} + prerelease: true + token: ${{ secrets.SNAPSHOT_RELEASE_TOKEN }} + files: | + gen/Firebird-*-android-${{ matrix.arch }}*.tar.gz diff --git a/builds/install/arch-specific/android/AfterUntar.sh b/builds/install/arch-specific/android/AfterUntar.sh index 8291ab89c4..92fb735934 100644 --- a/builds/install/arch-specific/android/AfterUntar.sh +++ b/builds/install/arch-specific/android/AfterUntar.sh @@ -1,6 +1,6 @@ -echo "This script will build arch-specific files, useful for firebird functionality" -echo "Press ENTER to continue" -read x +#!/bin/sh + +echo "This script builds arch-specific files, useful for firebird functionality" runAndCheckExit() { Msg=$1 diff --git a/builds/install/arch-specific/android/BuildFinalPackage.sh b/builds/install/arch-specific/android/BuildFinalPackage.sh new file mode 100755 index 0000000000..7be4679cb3 --- /dev/null +++ b/builds/install/arch-specific/android/BuildFinalPackage.sh @@ -0,0 +1,68 @@ +#!/bin/sh +set -e + +case $OSTYPE in + darwin*) + NDK_TOOLCHAIN_NAME=darwin-x86_64 + TAR_OPTS="--numeric-owner --uid=0 --gid=0" + FIND_EXEC_OPTS="-perm +0111" ;; + linux*) + NDK_TOOLCHAIN_NAME=linux-x86_64 + TAR_OPTS="--numeric-owner --owner=0 --group=0" + FIND_EXEC_OPTS="-executable" ;; +esac + +bits=${1} +[ -z "$bits" ] && bits=32 + +[ -z "$NDK_TOOLCHAIN" ] && NDK_TOOLCHAIN=$NDK/toolchains/llvm/prebuilt/$NDK_TOOLCHAIN_NAME +aStrip=${NDK_TOOLCHAIN}/bin/llvm-strip + +MakeVersion=gen/Make.Version +Build=`grep ^BuildNum ${MakeVersion}|awk '{print $3;}'` +Version=`grep ^FirebirdVersion ${MakeVersion}|awk '{print $3;}'` +InitialBaseName="Firebird-${Version}.${Build}-0-android-initial-arm${bits}" +InitialDebugTar="$InitialBaseName-withDebugSymbols.tar" +InitialDebugTarGz="$InitialDebugTar.gz" +Stripped=strip + +FinalRelease="Firebird-${Version}.${Build}-0-android-arm${bits}.tar.gz" +FinalDebug="Firebird-${Version}.${Build}-0-android-arm${bits}-withDebugSymbols.tar.gz" + +[ -z "$AndroidDevicePort" ] && AndroidDevicePort=5554 +AndroidDeviceName=emulator-$AndroidDevicePort +AndroidDir=/data/$InitialBaseName + +mkdir -p gen/Release +(cd gen; gunzip -k $InitialDebugTarGz) +(cd gen/Release; tar xvzf ../$InitialDebugTarGz) + +$ANDROID_HOME/platform-tools/adb -s $AndroidDeviceName shell "mkdir $AndroidDir" +$ANDROID_HOME/platform-tools/adb -s $AndroidDeviceName push gen/$InitialDebugTar $AndroidDir/ +$ANDROID_HOME/platform-tools/adb -s $AndroidDeviceName shell "(cd $AndroidDir && tar xvf $InitialDebugTar && cd firebird && ./AfterUntar.sh)" +$ANDROID_HOME/platform-tools/adb -s $AndroidDeviceName shell "(cd $AndroidDir/firebird/tests && ./common_test --log_level=all && ./libEngine13_test --log_level=all)" +$ANDROID_HOME/platform-tools/adb -s $AndroidDeviceName pull $AndroidDir/firebird/firebird.msg gen/Release/firebird/ +$ANDROID_HOME/platform-tools/adb -s $AndroidDeviceName pull $AndroidDir/firebird/security5.fdb gen/Release/firebird/ +$ANDROID_HOME/platform-tools/adb -s $AndroidDeviceName pull $AndroidDir/firebird/examples/empbuild/employe2.fdb gen/Release/firebird/examples/empbuild/ +$ANDROID_HOME/platform-tools/adb -s $AndroidDeviceName shell "(rm -rf $AndroidDir)" + +rm gen/$InitialDebugTar +cd gen/Release +rm -rf ${Stripped} + +rm -f firebird/{security.sql,employe2.sql,bin/build_file,AfterUntar.sh} + +tar $TAR_OPTS --exclude tests -czvf ../$FinalDebug firebird + +mkdir ${Stripped} +tar cf - firebird | (cd ${Stripped}; tar xvf -) +cd ${Stripped} +echo . +echo . +echo "Strip" +for file in `find firebird -type f $FIND_EXEC_OPTS -not -name "*.sh" -print` +do + ${aStrip} ${file} +done + +tar $TAR_OPTS --exclude tests -czvf ../../$FinalRelease firebird diff --git a/builds/install/arch-specific/android/BuildPackage.sh b/builds/install/arch-specific/android/BuildPackage.sh index 15dde5b893..996c07cba9 100755 --- a/builds/install/arch-specific/android/BuildPackage.sh +++ b/builds/install/arch-specific/android/BuildPackage.sh @@ -1,30 +1,36 @@ +#!/bin/sh +set -e + +case $OSTYPE in + darwin*) + NDK_TOOLCHAIN_NAME=darwin-x86_64 ;; + linux*) + NDK_TOOLCHAIN_NAME=linux-x86_64 ;; +esac + bits=${1} [ -z "$bits" ] && bits=32 + [ "$bits" = "32" ] && cross=arm-linux-androideabi [ "$bits" = "64" ] && cross=aarch64-linux-android [ -z "$cross" ] && echo "Invalid bits passed" && exit 1 -arm="" -[ "$bits" = "64" ] && arm=64 -[ -z "$NDK_TOOLCHAIN" ] && NDK_TOOLCHAIN=$NDK/toolchains/llvm/prebuilt/linux-x86_64 +[ -z "$NDK_TOOLCHAIN" ] && NDK_TOOLCHAIN=$NDK/toolchains/llvm/prebuilt/$NDK_TOOLCHAIN_NAME MakeVersion=gen/Make.Version Build=`grep ^BuildNum ${MakeVersion}|awk '{print $3;}'` Version=`grep ^FirebirdVersion ${MakeVersion}|awk '{print $3;}'` -Release="Firebird-${Version}.${Build}-0-android-arm${arm}.tar.gz" -Debug="Firebird-${Version}.${Build}-0-android-arm${arm}-withDebugSymbols.tar.gz" -Stripped=strip -aStrip=${NDK_TOOLCHAIN}/bin/llvm-strip +Release="Firebird-${Version}.${Build}-0-android-initial-arm${bits}.tar.gz" +Debug="Firebird-${Version}.${Build}-0-android-initial-arm${bits}-withDebugSymbols.tar.gz" fbRootDir=`pwd` runTar() { tarfile=${1} - tar cvfz ${tarfile} --exclude '*.a' --exclude tests firebird + tar cvfz ${tarfile} --exclude '*.a' --exclude '*.fdb' --exclude '*.msg' firebird } cd gen/Release -rm -rf ${Stripped} cp ${fbRootDir}/builds/install/arch-specific/android/AfterUntar.sh firebird chmod +x firebird/AfterUntar.sh cp ${fbRootDir}/src/dbs/security.sql firebird @@ -35,25 +41,3 @@ echo . echo . echo "Compress with deb-info" runTar ../${Debug} - -echo . -echo . -echo "Copy binaries" -mkdir ${Stripped} -tar cf - firebird | (cd ${Stripped}; tar xvf -) - - -cd ${Stripped} -echo . -echo . -echo "Strip" -for file in `find firebird -executable -type f -not -name "*.sh" -print` -do - ${aStrip} ${file} -done - -echo . -echo . -echo "Compress release" -runTar ../../${Release} -cd .. diff --git a/builds/install/arch-specific/android/CreateEmulator.sh b/builds/install/arch-specific/android/CreateEmulator.sh new file mode 100755 index 0000000000..a893d86cfd --- /dev/null +++ b/builds/install/arch-specific/android/CreateEmulator.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e + +echo no | $ANDROID_HOME/cmdline-tools/latest/bin/avdmanager create avd --name firebird-builder --package "system-images;android-30;google_apis;x86_64" diff --git a/builds/install/arch-specific/android/DeleteEmulator.sh b/builds/install/arch-specific/android/DeleteEmulator.sh new file mode 100755 index 0000000000..dcda78ee5d --- /dev/null +++ b/builds/install/arch-specific/android/DeleteEmulator.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e + +$ANDROID_HOME/cmdline-tools/latest/bin/avdmanager delete avd --name firebird-builder diff --git a/builds/install/arch-specific/android/StartEmulator.sh b/builds/install/arch-specific/android/StartEmulator.sh new file mode 100755 index 0000000000..94d0f080f3 --- /dev/null +++ b/builds/install/arch-specific/android/StartEmulator.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -e + +$ANDROID_HOME/emulator/emulator -no-window -avd firebird-builder -port 5554 & +$ANDROID_HOME/platform-tools/adb wait-for-device -s emulator-5554 +$ANDROID_HOME/platform-tools/adb -s emulator-5554 root diff --git a/builds/posix/make.android.arm64 b/builds/posix/make.android.arm64 index fcbf58608d..0e25b3e300 100644 --- a/builds/posix/make.android.arm64 +++ b/builds/posix/make.android.arm64 @@ -6,12 +6,10 @@ endif NDK_TOOLCHAIN ?= $(NDK)/toolchains/llvm/prebuilt/linux-x86_64 -CROSS_SYSROOT:=$(NDK_TOOLCHAIN)/sysroot CROSS_PREFIX24:=$(NDK_TOOLCHAIN)/bin/aarch64-linux-android24- CROSS_PREFIX:=$(NDK_TOOLCHAIN)/bin/llvm- -CROSS_FLAGS:=--sysroot=$(CROSS_SYSROOT) \ - -I$(CROSS_SYSROOT)/usr/include -I$(ROOT)/gen/cross +CROSS_FLAGS:=-I$(ROOT)/gen/cross CXX:=$(CROSS_PREFIX24)clang++ CC:=$(CROSS_PREFIX24)clang @@ -42,8 +40,9 @@ DEV_FLAGS=$(COMMON_FLAGS) $(WARN_FLAGS) CROSS_CONFIG=android.arm64 -LDFLAGS += --sysroot=$(CROSS_SYSROOT) -static-libstdc++ +LDFLAGS += -static-libstdc++ DroidLibs := -lm -ldl $(DECLIB) $(RE2LIB) $(I128LIB) +UDR_SUPPORT_LIBS := LINK_LIBS = $(DroidLibs) STATICLINK_LIBS = $(DroidLibs) diff --git a/builds/posix/make.android.arme b/builds/posix/make.android.arme index 4ce7b3715d..cbf56253f6 100644 --- a/builds/posix/make.android.arme +++ b/builds/posix/make.android.arme @@ -6,12 +6,10 @@ endif NDK_TOOLCHAIN ?= $(NDK)/toolchains/llvm/prebuilt/linux-x86_64 -CROSS_SYSROOT:=$(NDK_TOOLCHAIN)/sysroot CROSS_PREFIX24:=$(NDK_TOOLCHAIN)/bin/armv7a-linux-androideabi24- CROSS_PREFIX:=$(NDK_TOOLCHAIN)/bin/llvm- -CROSS_FLAGS:=--sysroot=$(CROSS_SYSROOT) \ - -I$(CROSS_SYSROOT)/usr/include -I$(ROOT)/gen/cross +CROSS_FLAGS:=-I$(ROOT)/gen/cross CXX:=$(CROSS_PREFIX24)clang++ CC:=$(CROSS_PREFIX24)clang @@ -42,13 +40,13 @@ DEV_FLAGS=$(COMMON_FLAGS) $(WARN_FLAGS) CROSS_CONFIG=android.arme -LDFLAGS += --sysroot=$(CROSS_PLATFORM) -static-libstdc++ +LDFLAGS += -static-libstdc++ DroidLibs := -lm -ldl $(DECLIB) $(RE2LIB) $(I128LIB) UDR_SUPPORT_LIBS := LINK_LIBS = $(DroidLibs) -SO_LINK_LIBS = $(DroidLibs) STATICLINK_LIBS = $(DroidLibs) +SO_LINK_LIBS = $(DroidLibs) EXE_LINK_OPTIONS= $(LDFLAGS) $(UNDEF_FLAGS) $(LIB_PATH_OPTS) $(LINK_EMPTY_SYMBOLS) -pie LIB_LINK_OPTIONS= $(LDFLAGS) -shared