gitlab-ci: fdroid build job fully handles reproducible builds

metadata/*/signatures/* are used for reproducible builds.  If those files
change, then `fdroid build` should run and try to build the app with those
signature files.  Also, `fdroid publish` needs to run to copy those
signature files to complete the test of reproducibility.  Builds that use
the `Binaries:` method are already verified as part of `fdroid build`.

fdroidserver!893
fdroidserver#891
fdroiddata!8816

.gitlab-ci.yml

@@ -152,6 +152,7 @@ fdroid build:
   artifacts:
     name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
     paths:
+      - repo/
       - unsigned/
       - tmp/
     when: always
@@ -168,12 +169,15 @@ fdroid build:
   script:
     - git fetch https://gitlab.com/fdroid/fdroiddata.git;
     - test -d build || mkdir build
-    - for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD -- metadata/*.yml`; do
+    - for f in
+          `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD -- metadata/*.yml`
+          `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD -- metadata/*/signatures`;
+      do
           diff=$(git diff FETCH_HEAD...HEAD -- $f);
           echo "$diff";
           test $(echo "$diff" | grep '^[+-] ' | grep -v '^+  *disable:' | wc -l) = 0 && continue;
           echo "$diff" | grep -E '^\+ *(NoSourceSince|Disabled):' && continue;
-          appid=`echo $f | sed -n -e 's,^metadata/\([^/][^/]*\)\.yml,\1,p'`;
+          appid=`echo $f | sed -E -n 's,^metadata/([^/][^/]*)(\.yml|/signatures/.*),\1,p'`;
           export CHANGED="$CHANGED $appid";
       done;
     - test -n "$(printf "$CHANGED" | tr -d '[:space:]')"
@@ -222,6 +226,16 @@ fdroid build:
           env TERM=$TERM
           env HOME=$VAGRANT_HOME
           fdroid"
+
+    # use fdroidserver test keystore as placeholder since `fdroid publish` requires it
+    - printf 'repo_keyalias\x3a sova\n' >> config.yml
+    - printf 'keystorepass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml
+    - printf 'keypass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml
+    - printf 'keydname\x3a "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"\n' >> config.yml
+    - keystore=$CI_PROJECT_DIR/fdroidserver/tests/keystore.jks
+    - printf "keystore\x3a $keystore\n" >> config.yml
+    - chown vagrant $keystore
+
     - for build in `./tools/find-changed-builds.py`; do
           set -x;
           apt-get install sudo;
@@ -229,6 +243,7 @@ fdroid build:
           chown -R vagrant $VAGRANT_HOME;
           $fdroid fetchsrclibs $build --verbose;
           $fdroid build --verbose --test --scan-binary --on-server --no-tarball $build;
+          ./tools/should-run-publish.py $build && (apt-get install sudo; $fdroid publish --verbose $build);
       done
 
 

tools/find-changed-builds.py

@@ -67,5 +67,16 @@ for appid in sorted(changed):
         for build in data['Builds']:
             to_build.append(build['versionCode'])
 
+    signatures_dir = 'metadata/%s/signatures/' % appid
+    diff = subprocess.check_output(
+        (
+            'git diff --name-only --no-color --diff-filter=d FETCH_HEAD...HEAD -- ' + signatures_dir
+        ).split(' ')
+    )
+    for f in diff.split():
+        vc = int(os.path.basename(os.path.dirname(f)))
+        if vc not in to_build:
+            to_build.append(vc)
+
     for vc in to_build:
         print('%s:%d' % (appid, vc), end=' ')

tools/should-run-publish.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+import glob
+import os
+import subprocess
+import sys
+import yaml
+from colorama import Fore, Style
+
+if len(sys.argv) != 2:
+    print(Fore.RED + ('ERROR: incorrect number of arguments') + Style.RESET_ALL)
+    exit(1)
+
+appid, versionCode = sys.argv[1].split(':')
+
+if not glob.glob('metadata/%s/signatures/%s/*' % (appid, versionCode)):
+    print(Fore.RED + ('ERROR: no signatures found') + Style.RESET_ALL)
+    exit(1)
+
+for f in glob.glob('tmp/%s_%s.apk' % (appid, versionCode)):
+    os.rename(f, f.replace('tmp/', 'unsigned/'))
+
+if not glob.glob('unsigned/%s_%s.apk' % (appid, versionCode)):
+    exit(1)