fdroiddata/.gitlab-ci.yml

include:
  - template: Security/SAST.gitlab-ci.yml

stages:
  - build
  - test
  - report

variables:
  pip: pip3 --timeout 100 --retries 10


checkupdates:
  image: registry.gitlab.com/fdroid/ci-images-checkupdates:latest
  except:
    - pipelines
    - schedules
    - triggers
  before_script:
    - |
      set -xe
      apt-get --allow-releaseinfo-change update
      apt-get -qy dist-upgrade
      cd ..
      rm -rf fdroidserver
      mkdir fdroidserver
      git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
      curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz | tar -xz --directory=fdroidserver --strip-components=1
      export PATH="$PWD/fdroidserver:$PATH"
      cd -      
  script: |
    set -xe
    if [ "$CI_PROJECT_NAMESPACE" = "fdroid" ]; then echo "not testing in fdroid"; exit 0; fi
    git fetch https://gitlab.com/fdroid/fdroiddata.git
    appids=$(git diff --name-only --diff-filter=d FETCH_HEAD...${CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_SHA:-HEAD} "metadata/*.yml" | sed -n -e 's,^metadata/\([^/][^/]*\)\.yml,\1,p')
    if [ -n "$appids" ]; then
      fdroid checkupdates --auto -v $appids
      git --no-pager diff --color=always --exit-code
    fi    

lint:
  image: registry.gitlab.com/fdroid/ci-images-base
  except:
    - pipelines
    - schedules
    - triggers
  before_script:
    - apt-get update
    - apt-get -qy dist-upgrade
    - apt-get -qy clean
    - rm -rf fdroidserver
    - mkdir fdroidserver
    - git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
    - curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz
        | tar -xz --directory=fdroidserver --strip-components=1
    - export PATH="$PWD/fdroidserver:$PATH"
  script:
    # if this is a merge request fork, then only check relevant apps
    - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
        git fetch https://gitlab.com/fdroid/fdroiddata.git;
        test -d build || mkdir build;
        files=`git diff --name-only --diff-filter=d FETCH_HEAD...${CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_SHA:-HEAD}`;
        for f in $files; do
           appid=`echo $f | sed -n -e 's,^metadata/\([^/][^/]*\)\.yml,\1,p'`;
           if [ -n "$appid" ]; then
             export CHANGED="$CHANGED $appid";
           fi;
        done;
        set -x;
        apt-get install python3-colorama yamllint;
        for f in $files; do [[ $f == *'.yml' ]] && yamllint "$f"; done;
        ./tools/check-git-repo-availability.py $files;
        set +x;
      fi
    - export EXITVALUE=0
    - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
    - fdroid lint -f $CHANGED || {
          set_error;
          printf "\nThese files have lint issues:\n";
          fdroid rewritemeta -l $CHANGED;
          printf "\nThese are the formatting issues:\n";
          fdroid rewritemeta $CHANGED;
          git --no-pager diff --color=always;
      }
    - apt-get -qy update
    - apt-get -qy install --no-install-recommends exiftool
    - find metadata/ -name '*.jp*g' -o -name '*.png' | xargs exiftool -all=
    - echo "these images have EXIF that must be stripped:"
    - git --no-pager diff --stat
    - git --no-pager diff --name-only --exit-code || set_error
    - ./tools/check-localized-metadata.py || set_error
    - ./tools/check-keyalias-collision.py || set_error
    - ./tools/check-metadata-summary-whitespace.py || set_error
    - ./tools/check-for-unattached-signatures.py || set_error
    - exit $EXITVALUE

schema validation:
  image: node:alpine
  except:
    - master@fdroid/fdroiddata
    - pipelines
    - schedules
    - triggers
  allow_failure:
    exit_codes: 128
  before_script:
    - apk add git
    - npm -g install ajv-cli
  script:
    - ajv compile -s schemas/*.json || exit 1
    - git fetch https://gitlab.com/fdroid/fdroiddata.git
    - set -xe
    - for file in
        $(git diff --name-only --diff-filter=d FETCH_HEAD...${CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_SHA:-HEAD} -- "metadata/*.yml");
      do
        ajv validate -s schemas/metadata.json -d "$file";
      done

trigger-issuebot:
  image: alpine
  except:
    - master@fdroid/fdroiddata
    - pipelines
    - schedules
    - triggers
  variables:
    GIT_DEPTH: "1"
  artifacts:
    name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
    paths:
      - logs/
    when: always
    expire_in: 1 week
  script:
    - mkdir logs
    - export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt
    - apk add --no-cache bash curl
    - ./tools/trigger-issuebot

schedule-issuebot:
  image: alpine
  only:
    variables:
      - $SCHEDULE_ISSUEBOT
  variables:
    GIT_DEPTH: "1"
  artifacts:
    name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
    paths:
      - logs/
    when: always
    expire_in: 1 week
  script:
    - mkdir logs
    - export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt
    - apk add --no-cache bash curl ca-certificates python3
    - python3 -m ensurepip
    - $pip install python-gitlab
    - ./tools/schedule-issuebot.py

checkupdates_trigger:
  only:
    refs:
      - schedules
    variables:
      - $CHECKUPDATES == "true"
  image: archlinux:latest
  before_script:
    - pacman -Sy --noconfirm parallel
    # workaround broken parallel version 20220222 not distributing jobs equally
    - pacman -U --noconfirm "https://archive.archlinux.org/packages/p/parallel/parallel-20220122-1-any.pkg.tar.zst"
  script:
    - parallel --verbose --delay 90 -X --jobs 10 'curl -X POST -F "token=$CI_JOB_TOKEN" -F "ref=master" -F "variables[CHECKUPDATES]=true" -F "variables[CHECKUPDATES_APPIDS]= {/.} " https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/trigger/pipeline' ":::" metadata/*.yml

checkupdates_runner:
  image: registry.gitlab.com/fdroid/ci-images-checkupdates:latest
  tags:
    - checkupdates-runner
  only:
    refs:
      - pipelines
    variables:
      - $CHECKUPDATES == "true"
  before_script:
    - rm -rf fdroidserver
    - mkdir fdroidserver
    - git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
    - curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz
        | tar -xz --directory=fdroidserver --strip-components=1
    - export PATH="$PWD/fdroidserver:$PATH"
    - git config --global user.email "fdroidci@bubu1.eu"
    - git config --global user.name "F-Droid checkupdates bot"
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - cp "${GITLAB_KNOWN_HOSTS}" ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
    - eval $(ssh-agent -s)
    - echo "${CHECKUPDATES_SSH_DEPLOY_KEY}" | tr -d '\r' | ssh-add -
    - url_host=$(echo "${CI_REPOSITORY_URL}" | sed -e 's|https\?://gitlab-ci-token:.*@|ssh://git@|g')
    - git remote set-url --push origin "${url_host}"
    # reset repo to origin state before adding new commits
    - git rebase --abort || true
    - rm -fr ".git/rebase-apply"
    - git checkout master || true
    - git reset --hard origin/master || true
  script:
    - fdroid checkupdates --allow-dirty --auto --commit ${CHECKUPDATES_APPIDS} 2>&1 | tee /tmp/out || true
    - git pull --rebase origin master
    # when two jobs try to push at the same time they occasionally fail, so try one more time if it fails
    - git push origin HEAD:master || (git pull --rebase origin master && git push origin HEAD:master)
    - echo "============== Summary ====================="
    - 'grep -v "INFO: Processing" /tmp/out || true'

fdroid build:
  image: registry.gitlab.com/fdroid/fdroidserver:buildserver
  artifacts:
    name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
    paths:
      - repo/
      - unsigned/
      - tmp/
    when: always
    expire_in: 1 month
  except:
    - master@fdroid/fdroiddata
    - pipelines
    - schedules
    - triggers
  cache:
    key: "$CI_JOB_NAME"
    paths:
      - .gradle
  script:
    - git fetch https://gitlab.com/fdroid/fdroiddata.git;
    - test -d build || mkdir build
    - for f in
          `git diff --name-only --diff-filter=d FETCH_HEAD...${CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_SHA:-HEAD} -- metadata/*.yml`
          `git diff --name-only --diff-filter=d FETCH_HEAD...${CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_SHA:-HEAD} -- metadata/*/signatures`;
      do
          diff=$(git diff FETCH_HEAD...${CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_SHA:-HEAD} -- $f);
          echo "$diff";
          test $(echo "$diff" | grep '^[+-] ' | grep -v '^+  *disable:' | wc -l) = 0 && continue;
          echo "$diff" | grep -E '^\+ *(NoSourceSince|Disabled):' && continue;
          appid=`echo $f | sed -E -n 's,^metadata/([^/][^/]*)(\.yml|/signatures/.*),\1,p'`;
          export CHANGED="$CHANGED $appid";
      done;
    - test -n "$(printf "$CHANGED" | tr -d '[:space:]')"
          || { echo "no packages need processing, exiting"; exit 0; }

    - apt-get update
    - apt-get dist-upgrade
    - apt-get clean

    - test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh

    # TODO remove sdkmanager install once it is included in the buildserver image
    - git clone --depth=1 --branch=0.5.2 https://gitlab.com/fdroid/sdkmanager.git
    - git -C sdkmanager checkout -B master 59374827baed7020c9761d9c8f51dcf9b490ba28
    - pip3 install -e sdkmanager
    - rm -rf $ANDROID_HOME/tools  # TODO remove once sdkmanager can upgrade installed packages
    - sdkmanager "tools" "platform-tools" "build-tools;31.0.0"

    - test -d $fdroidserver || mkdir $fdroidserver
    - git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
    - curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz
          | tar -xz --directory=$fdroidserver --strip-components=1

    - for d in logs tmp unsigned $home_vagrant/build $home_vagrant/.android; do
          test -d $d || mkdir $d;
          chown -R vagrant $d;
      done
    - ln -s $home_vagrant/build $CI_PROJECT_DIR/build

    - export GRADLE_USER_HOME=$home_vagrant/.gradle
    # each `fdroid build --on-server` run expects sudo, then uninstalls it
    - export fdroid="sudo --preserve-env --user vagrant
          env PATH=$fdroidserver:$PATH
          env PYTHONPATH=$fdroidserver:$fdroidserver/examples
          env PYTHONUNBUFFERED=true
          env TERM=$TERM
          env HOME=$home_vagrant
          fdroid"

    # use fdroidserver test keystore as placeholder since `fdroid publish` requires it
    - printf 'repo_keyalias\x3a sova\n' >> config.yml
    - printf 'keystorepass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml
    - printf 'keypass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml
    - printf 'keydname\x3a "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"\n' >> config.yml
    - keystore=$fdroidserver/tests/keystore.jks
    - printf "keystore\x3a $keystore\n" >> config.yml
    - chown vagrant $keystore

    - for build in `./tools/find-changed-builds.py`; do
          set -x;
          apt-get install sudo;
          chown -R vagrant $home_vagrant;
          $fdroid fetchsrclibs $build --verbose;
          $fdroid build --verbose --test --scan-binary --on-server --no-tarball $build;
          rm -rf "build/${build%:*}";
          ./tools/build-contains-signatures.py $build || continue;
          apt-get install sudo;
          $fdroid publish --verbose $build;
      done

    - ./tools/audit-gradle.py $CHANGED;


# issuebot needs secrets to run, so it has to run under the 'fdroid'
# group, therefore needs the trigger without secrets, there would be
# no support for virustotal, github downloads, exodus privacy checks,
# etc.  That means it has to be triggered from the lint: job, which
# runs in the fork's CI. The trigger-issuebot job adds the env var
# FROM_CI_MERGE_REQUEST_IID which is required to have this job be
# triggered.
#
# This job has to be called 'pages' because it deploys the JSON API
# to GitLab Pages.
pages:
  image: registry.gitlab.com/fdroid/fdroidserver:buildserver
  only:
    refs:
      - branches
    variables:
      - $FROM_CI_MERGE_REQUEST_IID
  artifacts:
    name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_JOB_ID}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
    paths:
      - metadata/
      - public/
      - repo/index-v1.json
      - repo/index.xml
      - tmp/apkcache.json
      - unsigned/
    when: always
  # needs lots of git history since it has to compare the merge request to current master
  variables:
    GIT_DEPTH: "5000"
  script:
    - apt-get update
    - apt-get dist-upgrade
    - apt-get install python3-venv

    - test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh
    - test -d $fdroidserver || mkdir $fdroidserver
    - git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
    - curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz
        | tar -xz --directory=$fdroidserver --strip-components=1
    - export PATH="$fdroidserver:$PATH"
    - export PYTHONPATH="$fdroidserver:$fdroidserver/examples"
    - export PYTHONUNBUFFERED=true

    - export GRADLE_USER_HOME=$PWD/.gradle
    - rm -rf $GRADLE_USER_HOME/fdroid
    - mkdir -p $GRADLE_USER_HOME/fdroid
    - git ls-remote https://gitlab.com/fdroid/gradle-plugins.git master
    - curl --silent https://gitlab.com/fdroid/gradle-plugins/-/archive/master/gradle-plugins-master.tar.gz
        | tar -xz --directory=$GRADLE_USER_HOME/fdroid --strip-components=1

    - git ls-remote https://gitlab.com/fdroid/issuebot.git master
    - curl --silent https://gitlab.com/fdroid/issuebot/-/archive/master/issuebot-master.tar.gz
        | tar -xz --strip-components=1
    - pyvenv --system-site-packages --clear issuebot-env
    - . issuebot-env/bin/activate
    - pip3 install wheel # get rid of the bdist_wheel error messages
    - pip3 install python-gitlab pygithub
    - ./issuebot.py

    # git_stats used to run here, redirect to new location
    - echo '<html><head><meta http-equiv="refresh" content="0;URL=https://fdroid.gitlab.io/"></head></html>'
        > public/index.html


check_git_repos:
  image: debian:buster-slim
  stage: test
  only:
    refs:
      - schedules
    variables:
      - $CHECK_GIT_REPO == "true"
  artifacts:
    when: on_failure
    expire_in: 1 month
    paths:
      - public
  script:
    - apt-get update
    - apt-get -qy install --no-install-recommends ca-certificates git python3-colorama python3-yaml
    - tools/check-git-repo-availability.py || export EXITVALUE=1
    - test -d public || mkdir public
    - cp `git status | grep -Eo 'metadata/.*\.yml'` public/ || true
    - exit $EXITVALUE

sast:
  stage: report
  variables:
    MOBSF_API_KEY: key
    SAST_EXPERIMENTAL_FEATURES: "true"
    SAST_EXCLUDED_ANALYZERS:
      bandit, brakeman, eslint, flawfinder, gosec, kubesec,
      nodejs-scan, phpcs-security-audit, pmd-apex, security-code-scan, semgrep, sobelow,
      spotbugs

    # Make sure the apk is searched before the manifest, so only apks are scanned
    ANALYZER_TARGET_DIR: $CI_PROJECT_DIR/tmp/
  rules:
    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
      when: never
  allow_failure: true

mobsf-ios-sast:
  rules:
    - when: never