fdroiddata/.gitlab-ci.yml

workflow:
  rules:
    # required to enable merge request pipelines: https://docs.gitlab.com/ee/ci/pipelines/merge_request_pipelines.html#prerequisites
    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_ENABLE_MERGE_REQUEST_PIPELINES != "false"
    # to prevent duplicate pipelines in MRs: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" && $CI_ENABLE_MERGE_REQUEST_PIPELINES != "false"
      when: never
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH

stages:
  - prepare
  - build
  - test
  - report
  - deploy

variables:
  pip: pip3 --timeout 100 --retries 10

  GIT_DEPTH:
    description: needs lots of git history since it has to compare the merge request to current master
    value: "5000"

  CI_ENABLE_MERGE_REQUEST_PIPELINES:
    description: push with `git push -o ci.variable="CI_ENABLE_MERGE_REQUEST_PIPELINES=false"` to disable merge request pipelines (e.g., to use self-hosted runners)
    value: "true"

.install_fdroid_server: &install_fdroid_server
  - rm -rf $fdroidserver
  - mkdir $fdroidserver
  - git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
  - curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz
        | tar -xz --directory=$fdroidserver --strip-components=1
  - export PATH="$fdroidserver:$PATH"
  - export PYTHONPATH="$fdroidserver:$fdroidserver/examples"
  - export PYTHONUNBUFFERED=true

.install_fdroid_server_deb: &install_fdroid_server_deb
  - apt-get --allow-releaseinfo-change update
  - apt-get -qy dist-upgrade
  - echo "locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8" | debconf-set-selections
  - apt-get install -qy --no-install-recommends fdroidserver apksigner mercurial git git-svn brz python3-launchpadlib locales curl

.get_target_source_refs: &get_target_source_refs
  - |
    if [ "$CI_PROJECT_PATH" = "fdroid/fdroiddata" ] && [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
        export TARGET_REF="${CI_COMMIT_BEFORE_SHA}"
        export SOURCE_REF="${CI_COMMIT_SHA}"
    else
        git remote add upstream https://gitlab.com/fdroid/fdroiddata.git || true
        git fetch upstream ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-master}
        export TARGET_REF=$(git merge-base HEAD upstream/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-master})
        export SOURCE_REF="${CI_COMMIT_SHA}"
    fi;    

.get_changed_builds: &get_changed_builds
  - *get_target_source_refs
  - echo $TARGET_REF
  - |
    for f in $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*.yml) $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*/signatures); do
        diff=$(git diff "${TARGET_REF}...${SOURCE_REF}" -- "$f")
        echo "$diff"
        test $(echo "$diff" | perl -wnle '/^[+-](( +-)|( *\w))/ and print' | grep -v -c '^+ *disable:') = 0 && continue
        echo "$diff" | grep -E '^\+ *(NoSourceSince|Disabled):' && continue
        appid=$(echo "$f" | sed -E -n 's,^metadata/([^/][^/]*)(\.yml|/signatures/.*),\1,p')
        export CHANGED="$CHANGED $appid"
    done    

.get_changed_apps: &get_changed_apps
  - *get_target_source_refs
  - echo $TARGET_REF
  - |
    for f in $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*.yml) $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*/signatures); do
        diff=$(git diff "${TARGET_REF}...${SOURCE_REF}" -- "$f")
        appid=$(echo "$f" | sed -E -n 's,^metadata/([^/][^/]*)(\.yml|/signatures/.*),\1,p')
        export CHANGED="$CHANGED $appid"
    done    

.app_verification_rules:
  rules: &app_verification_rules
    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger"
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_ENABLE_MERGE_REQUEST_PIPELINES != "false"
      when: never
    - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      # only works for merge request pipelines: https://docs.gitlab.com/ee/ci/jobs/job_control.html#jobs-or-pipelines-run-unexpectedly-when-using-changes
      changes:
        - metadata/**/*
        - srclibs/**/*
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH

checkupdates:
  stage: test
  needs: []
  tags:
    - saas-linux-large-amd64
  image: debian:bookworm-slim
  rules: *app_verification_rules
  before_script:
    - *install_fdroid_server_deb
    - export fdroidserver=../fdroidserver
    - *install_fdroid_server
  script:
    - |
      set -xe      
    - *get_changed_builds
    - |
      if [ -n "$CHANGED" ]; then
        fdroid checkupdates --auto -v $CHANGED
        git --no-pager diff --color=always --exit-code
      fi      

git redirect:
  stage: test
  needs: []
  image: debian:bookworm-slim
  rules: *app_verification_rules
  script:
    - apt-get -qy update
    - apt-get -qy dist-upgrade
    - apt-get -qy install --no-install-recommends ca-certificates git python3 python3-yaml
    - *get_changed_apps
    - test -n "$CHANGED" || exit 0
    - tools/rewrite-git-redirects.py $CHANGED
    - git --no-pager diff --color=always --exit-code || {
          printf "\x1b[31mERROR git URLs should be direct, not redirects!\x1b[0m\n";
          exit 1;
      }

fdroid lint:
  stage: test
  needs: []
  image: debian:bookworm-slim
  rules: *app_verification_rules
  before_script:
    - *install_fdroid_server_deb
    - export fdroidserver=$PWD/fdroidserver
    - *install_fdroid_server
  script:
    - cp tools/lint_config.yml config.yml
    - *get_changed_apps
    - export EXITVALUE=0
    - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
    - |
      if [ -n "$CHANGED" ]; then
        fdroid lint -f $CHANGED || {
            set_error;
            printf "\nThese files have lint issues:\n";
            fdroid rewritemeta -l $CHANGED;
            printf "\nThese are the formatting issues:\n";
            fdroid rewritemeta $CHANGED;
            git --no-pager diff --color=always;
        }
      fi      
    - exit $EXITVALUE


fdroid lint config:
  stage: test
  needs: []
  image: debian:bookworm-slim
  before_script:
    - apt-get -qy update
    - apt-get -qy dist-upgrade
    - apt-get -qy install --no-install-recommends
        ca-certificates
        curl
        git
        python3-asn1crypto
        python3-defusedxml
        python3-git
        python3-pycountry
        python3-qrcode
        python3-requests
        python3-ruamel.yaml
        yamllint
    - export fdroidserver=$CI_PROJECT_DIR/fdroidserver
    - *install_fdroid_server
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push"
      changes:
        paths:
          - .gitlab-ci.yml
          - config.yml
          - config/*.yml
          - config/*/*.yml
  script:
    - fdroid lint config/*.yml config/*/*.yml
    # TODO once config.yml is added to fdroiddata, add it to the line above
    - test -e config.yml || exit 0
    - fdroid lint config.yml


lint:
  stage: test
  needs: []
  image: debian:bookworm-slim
  rules: *app_verification_rules
  script:
    - apt-get -qy update
    - apt-get -qy dist-upgrade
    - apt-get -qy install --no-install-recommends exiftool git python3 python3-yaml
    - export EXITVALUE=0
    - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }

    - find metadata/ -name '*.jp*g' -o -name '*.png' | xargs exiftool -all=
    - echo "these images have EXIF that must be stripped:"
    - git --no-pager diff --stat
    - git --no-pager diff --name-only --exit-code || set_error
    - ./tools/check-localized-metadata.py || set_error
    - ./tools/check-keyalias-collision.py || set_error
    - ./tools/check-metadata-summary-whitespace.py || set_error
    - ./tools/check-for-unattached-signatures.py || set_error
    - ./tools/make-summary-translatable.py || set_error

    - exit $EXITVALUE

schema validation:
  stage: test
  needs: []
  image: python
  rules: *app_verification_rules
  before_script:
    - pip install check-jsonschema
  script:
    - check-jsonschema --check-metaschema schemas/metadata.json || exit 1
    - *get_target_source_refs
    - git diff --exit-code --name-only "${TARGET_REF}...${SOURCE_REF}" schemas/metadata.json
          || { check-jsonschema --schemafile schemas/metadata.json metadata/*.yml || exit 1; exit; }
    - set -xe
    - *get_changed_apps
    - if [ -z "$CHANGED" ]; then exit; fi
    - echo $CHANGED | sed -En 's|([^ ]*)|metadata/\1.yml|gp' | xargs check-jsonschema --schemafile schemas/metadata.json

trigger-issuebot:
  stage: prepare
  needs: []
  when: manual
  image: alpine
  variables:
    GIT_DEPTH: "1"
  artifacts:
    name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
    paths:
      - logs/
    when: always
    expire_in: 1 week
  script:
    - mkdir logs
    - export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt
    - apk add --no-cache bash curl
    - ./tools/trigger-issuebot

schedule-issuebot:
  stage: test
  needs: []
  image: debian:bookworm-slim
  only:
    variables:
      - $SCHEDULE_ISSUEBOT
  variables:
    GIT_DEPTH: "1"
  artifacts:
    name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
    paths:
      - logs/
    when: always
    expire_in: 1 week
  script:
    - mkdir logs
    - export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt
    - apt-get update
    - apt-get -qy install --no-install-recommends bash curl ca-certificates python3-gitlab
    - ./tools/schedule-issuebot.py

checkupdates_runner:
  stage: prepare
  image: debian:bookworm-slim
  variables:
    GIT_DEPTH: "1"
  parallel: 10
  tags:
    - fdroid
    - pep.security
  rules:
    - if: $CI_PIPELINE_SOURCE == 'schedule' && $CHECKUPDATES == 'true'
  before_script:
    - *install_fdroid_server_deb
    - export fdroidserver=../fdroidserver
    - *install_fdroid_server
    - apt-get install -y openssh-client
    - git config --global user.email "fdroidci@bubu1.eu"
    - git config --global user.name "F-Droid checkupdates bot"

    # gitlab.com was still vulnerable to https://terrapin-attack.com/ when this was added
    - printf 'Ciphers -chacha20-poly1305@openssh.com,*-cbc\nMACs -*etm*,*-sha1*\n'
        > /etc/ssh/ssh_config.d/0-terrapin-workaround.conf

    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - cp "${GITLAB_KNOWN_HOSTS}" ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
    - eval $(ssh-agent -s)
    - echo "${CHECKUPDATES_SSH_DEPLOY_KEY}" | tr -d '\r' | ssh-add -
    - url_host=$(echo "${CI_REPOSITORY_URL}" | sed -e 's|https\?://gitlab-ci-token:.*@|ssh://git@|g')
    - git remote set-url --push origin "${url_host}"
    # reset repo to origin state before adding new commits
    - git rebase --abort || true
    - rm -fr ".git/rebase-apply"
    # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/29187
    - rm -f ".git/logs/HEAD.lock"
    - git checkout master || true
    - git reset --hard origin/master || true
  script:
    # Get app IDs in this batch
    - |
      metadata_files=(metadata/*.yml)
      metadata_files_count=${#metadata_files[@]}
      batch_size=$((metadata_files_count / CI_NODE_TOTAL + 1))
      metadata_files_batch=("${metadata_files[@]:$(((CI_NODE_INDEX - 1) * batch_size)):$batch_size}")

      declare -a CHECKUPDATES_APPIDS
      for file in "${metadata_files_batch[@]}"; do
        filename="${file##*/}"
        filename_without_extension="${filename%.*}"
        CHECKUPDATES_APPIDS=("${CHECKUPDATES_APPIDS[@]}" "$filename_without_extension")
      done

      export CHECKUPDATES_APPIDS      

    - echo "$metadata_files_count"
    - echo "$batch_size"
    - echo "${metadata_files_batch[@]}"
    - echo "${#metadata_files_batch[@]}"
    - echo "${CHECKUPDATES_APPIDS[@]}"
    - echo "${#CHECKUPDATES_APPIDS[@]}"

    - fdroid checkupdates --allow-dirty --auto --commit "${CHECKUPDATES_APPIDS[@]}" 2>&1 | tee /tmp/out || true
    # when two jobs try to push at the same time they occasionally fail, so try it again
    - while ! git push origin HEAD:master; do git pull --rebase --strategy-option=ours origin master; done
    - echo "============== Summary ====================="
    - 'grep -v "INFO: Processing" /tmp/out || true'

# This job is should be as close as possible to the production
# buildserver.  It should not include custom setup or methods, except
# when there is no other way to make this job work.  The docker image
# is created using the same provisioning as the producion buildserver,
# via the "docker" job in fdroid/fdroidserver.
#
# This job is tied to the buildserver.  Changes must be reviewed by
# the buildserver maintainers.
fdroid build:
  stage: build
  needs: []
  tags:
    - saas-linux-large-amd64
  # Some env vars are setup in https://gitlab.com/fdroid/fdroidserver/-/blob/master/buildserver/setup-env-vars
  # home_vagrant=/home/vagrant
  image: registry.gitlab.com/fdroid/fdroidserver:buildserver-bookworm
  artifacts:
    name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
    paths:
      - repo/
      - unsigned/
      - tmp/
    when: always
    expire_in: 1 month
  rules: *app_verification_rules
  cache:
    key: "$CI_JOB_NAME"
    paths:
      - .gradle
  variables:
    ANDROID_HOME: /opt/android-sdk
    ANDROID_SDK_ROOT: ${ANDROID_HOME}
  script:
    - test -d build || mkdir build
    - *get_changed_builds
    - test -n "$(printf "$CHANGED" | tr -d '[:space:]')"
          || { echo "no packages need processing, exiting"; exit 0; }

    - apt-get update
    - apt-get dist-upgrade

    - test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh

    # These packages are needed to make this env like the production buildserver.
    - sdkmanager "platform-tools" "build-tools;31.0.0"

    - *install_fdroid_server

    # Set up git config something like the production buildserver. The
    # difference is that in production, this is installed on the
    # buildserver VM host, while here it is basically in the
    # buildserver VM.
    # https://gitlab.com/fdroid/fdroid-bootstrap-buildserver/-/merge_requests/30
    - curl --silent 'https://gitlab.com/fdroid/fdroid-bootstrap-buildserver/-/raw/master/roles/production_hardening/files/gitconfig' >> $CI_PROJECT_DIR/../.gitconfig

    - for d in logs tmp unsigned $home_vagrant/.android $home_vagrant/.gradle $home_vagrant/metadata; do
          test -d $d || mkdir $d;
          chown -R vagrant $d;
      done
    # make sure .gradle stores inside $CI_PROJECT_DIR to make it cacheable
    - ln -s $home_vagrant/.gradle $CI_PROJECT_DIR/.gradle
    - ln -s $CI_PROJECT_DIR/tmp $home_vagrant/tmp
    - ln -s $CI_PROJECT_DIR/srclibs $home_vagrant/srclibs

    - export GRADLE_USER_HOME=$home_vagrant/.gradle
    # each `fdroid build --on-server` run expects sudo, then uninstalls it
    - export fdroid="sudo --preserve-env --user vagrant
          env PATH=$fdroidserver:$PATH
          env PYTHONPATH=$fdroidserver:$fdroidserver/examples
          env PYTHONUNBUFFERED=true
          env TERM=$TERM
          env HOME=$home_vagrant
          fdroid"

    # use fdroidserver test keystore as placeholder since `fdroid publish` requires it
    - printf 'repo_keyalias\x3a sova\n' >> config.yml
    - printf 'keystorepass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml
    - printf 'keypass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml
    - printf 'keydname\x3a "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"\n' >> config.yml
    - keystore=$fdroidserver/tests/keystore.jks
    - printf "keystore\x3a $keystore\n" >> config.yml

    - APPS="$(./tools/find-changed-builds.py)"
    - for build in $APPS; do
          set -x;
          apt-get install sudo;
          cp -R $CI_PROJECT_DIR/build $home_vagrant/build;
          appid=${build%:*};
          [ -d metadata/$appid ] && cp -R metadata/$appid $home_vagrant/metadata;
          cp -R metadata/$appid.yml $home_vagrant/metadata;
          chown -R vagrant $home_vagrant $CI_PROJECT_DIR;
          pushd $home_vagrant;
          ln -s $CI_PROJECT_DIR $home_vagrant/fdroiddata;
          ln -s $CI_PROJECT_DIR/../.gitconfig $home_vagrant/.gitconfig;
          $fdroid fetchsrclibs $build --verbose;
          rm $home_vagrant/fdroiddata $home_vagrant/.gitconfig;
          $fdroid build --verbose --test --refresh-scanner --scan-binary --on-server --no-tarball $build;
          popd;
          rm -rf $home_vagrant/build || true;
          rm -rf $ANDROID_HOME/ndk || true;
          apt-get install -y openjdk-17-jdk-headless;
          update-alternatives --set java /usr/lib/jvm/java-17-openjdk-amd64/bin/java;
          ./tools/build-contains-signatures.py $build || continue;
          apt-get install sudo;
          $fdroid publish --verbose --error-on-failed $build;
      done

    - ./tools/audit-gradle.py $CHANGED;


# issuebot needs secrets to run, so it has to run under the 'fdroid'
# group, therefore needs the trigger without secrets, there would be
# no support for virustotal, github downloads, exodus privacy checks,
# etc.  That means it has to be triggered from the lint: job, which
# runs in the fork's CI. The trigger-issuebot job adds the env var
# FROM_CI_MERGE_REQUEST_IID which is required to have this job be
# triggered.
#
# This job has to be called 'pages' because it deploys the JSON API
# to GitLab Pages.
pages:
  stage: deploy
  tags:
    - saas-linux-large-amd64
  image: registry.gitlab.com/fdroid/fdroidserver:buildserver-bookworm
  only:
    refs:
      - branches
    variables:
      - $FROM_CI_MERGE_REQUEST_IID
  artifacts:
    name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_JOB_ID}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
    paths:
      - metadata/
      - public/
      - repo/index-v1.json
      - repo/index.xml
      - tmp/apkcache.json
      - unsigned/
    when: always
  script:
    - apt-get update
    - apt-get dist-upgrade
    - apt-get install
         python3-github
         python3-gitlab
         python3-defusedxml
         python3-javaproperties
         python3-requests-cache
         python3-venv
         python3-wheel

    - test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh
    - *install_fdroid_server

    - export GRADLE_USER_HOME=$PWD/.gradle
    - rm -rf $GRADLE_USER_HOME/fdroid
    - mkdir -p $GRADLE_USER_HOME/fdroid
    - git ls-remote https://gitlab.com/fdroid/gradle-plugins.git master
    - curl --silent https://gitlab.com/fdroid/gradle-plugins/-/archive/master/gradle-plugins-master.tar.gz
        | tar -xz --directory=$GRADLE_USER_HOME/fdroid --strip-components=1

    - git ls-remote https://gitlab.com/fdroid/issuebot.git master
    - curl --silent https://gitlab.com/fdroid/issuebot/-/archive/master/issuebot-master.tar.gz
        | tar -xz --strip-components=1
    - python3 -m venv --system-site-packages --clear issuebot-env
    - . issuebot-env/bin/activate
    - ./issuebot.py

    # git_stats used to run here, redirect to new location
    - echo '<html><head><meta http-equiv="refresh" content="0;URL=https://fdroid.gitlab.io/"></head></html>'
        > public/index.html


check_git_repos:
  image: debian:bookworm-slim
  stage: test
  needs: []
  only:
    refs:
      - schedules
    variables:
      - $CHECK_GIT_REPO == "true"
  artifacts:
    when: on_failure
    expire_in: 1 month
    paths:
      - public
  script:
    - apt-get update
    - apt-get -qy install --no-install-recommends ca-certificates git python3-colorama python3-yaml
    - tools/check-git-repo-availability.py || export EXITVALUE=1
    - test -d public || mkdir public
    - cp `git status | grep -Eo 'metadata/.*\.yml'` public/ || true
    - exit $EXITVALUE