variables: pip: pip3 --timeout 100 --retries 10 lint: image: registry.gitlab.com/fdroid/ci-images-base except: - pipelines - schedules - triggers before_script: - apt-get update - apt-get -qy dist-upgrade - rm -rf fdroidserver - mkdir fdroidserver - git ls-remote https://gitlab.com/fdroid/fdroidserver.git master - curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz | tar -xz --directory=fdroidserver --strip-components=1 - export PATH="$PWD/fdroidserver:$PATH" - touch config.py script: # if this is a merge request fork, then only check relevant apps - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then git fetch https://gitlab.com/fdroid/fdroiddata.git; test -d build || mkdir build; files=`git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; for f in $files; do appid=`echo $f | sed -n -e 's,^metadata/\([^/][^/]*\)\.yml,\1,p'`; if [ -n "$appid" ]; then export CHANGED="$CHANGED $appid"; testcmd="git -C build clone `sed -En 's,^Repo\W +(https?://),\1u:p@,p' $f`"; fi; done; set -x; apt-get install python3-colorama yamllint; for f in $files; do [[ $f == *'.yml' ]] && yamllint "$f"; done; ./tools/check-git-repo-availability.py $files; ./tools/audit-gradle.py $CHANGED; set +x; fi - export EXITVALUE=0 - fdroid lint -f $CHANGED || { export EXITVALUE=1; printf "\nThese files have lint issues:\n"; fdroid rewritemeta -l $CHANGED; printf "\nThese are the formatting issues:\n"; fdroid rewritemeta $CHANGED; git --no-pager diff --color=always; } - apt-get -qy update - apt-get -qy install --no-install-recommends exiftool - find metadata/ -name '*.jp*g' -o -name '*.png' | xargs exiftool -all= - echo "these images have EXIF that must be stripped:" - git --no-pager diff --stat - git --no-pager diff --name-only --exit-code || export EXITVALUE=1 - ./tools/check-localized-metadata.py || export EXITVALUE=1 - ./tools/check-keyalias-collision.py || export EXITVALUE=1 - ./tools/check-metadata-summary-whitespace.py || export EXITVALUE=1 - ./tools/check-for-unattached-signatures.py || export EXITVALUE=1 - exit $EXITVALUE trigger-issuebot: image: alpine except: - master@fdroid/fdroiddata - pipelines - schedules - triggers variables: GIT_DEPTH: "1" artifacts: name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}" paths: - logs/ when: always expire_in: 1 week script: - mkdir logs - export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt - apk add --no-cache bash curl - ./tools/trigger-issuebot schedule-issuebot: image: alpine only: variables: - $SCHEDULE_ISSUEBOT variables: GIT_DEPTH: "1" artifacts: name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}" paths: - logs/ when: always expire_in: 1 week script: - mkdir logs - export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt - apk add --no-cache bash curl ca-certificates python3 - python3 -m ensurepip - $pip install python-gitlab - ./tools/schedule-issuebot.py checkupdates_trigger: only: refs: - schedules variables: - $CHECKUPDATES == "true" image: archlinux:latest before_script: - pacman -Sy --noconfirm parallel script: - parallel --verbose --delay 90 -X --jobs 10 'curl -X POST -F "token=$CI_JOB_TOKEN" -F "ref=master" -F "variables[CHECKUPDATES]=true" -F "variables[CHECKUPDATES_APPIDS]= {/.} " https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/trigger/pipeline' ":::" metadata/*.yml checkupdates_runner: image: registry.gitlab.com/fdroid/ci-images-checkupdates:latest tags: - checkupdates-runner only: refs: - pipelines variables: - $CHECKUPDATES == "true" before_script: - rm -rf fdroidserver - mkdir fdroidserver - git ls-remote https://gitlab.com/fdroid/fdroidserver.git master - curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz | tar -xz --directory=fdroidserver --strip-components=1 - export PATH="$PWD/fdroidserver:$PATH" - touch config.py - git config --global user.email "fdroidci@bubu1.eu" - git config --global user.name "F-Droid checkupdates bot" - mkdir -p ~/.ssh - chmod 700 ~/.ssh - cp "${GITLAB_KNOWN_HOSTS}" ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts - eval $(ssh-agent -s) - echo "${CHECKUPDATES_SSH_DEPLOY_KEY}" | tr -d '\r' | ssh-add - - url_host=$(echo "${CI_REPOSITORY_URL}" | sed -e 's|https\?://gitlab-ci-token:.*@|ssh://git@|g') - git remote set-url --push origin "${url_host}" script: - fdroid checkupdates --allow-dirty --auto --commit ${CHECKUPDATES_APPIDS} - git pull --rebase origin master # when two jobs try to push at the same time they occasionally fail, so try one more time if it fails - git push origin HEAD:master || (git pull --rebase origin master && git push origin HEAD:master) fdroid build: image: registry.gitlab.com/fdroid/ci-images-client:latest artifacts: name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}" paths: - repo/ - unsigned/ - tmp/ when: always expire_in: 1 month except: - master@fdroid/fdroiddata - pipelines - schedules - triggers cache: key: "$CI_JOB_NAME" paths: - .gradle script: - git fetch https://gitlab.com/fdroid/fdroiddata.git; - test -d build || mkdir build - for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD -- metadata/*.yml` `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD -- metadata/*/signatures`; do diff=$(git diff FETCH_HEAD...HEAD -- $f); echo "$diff"; test $(echo "$diff" | grep '^[+-] ' | grep -v '^+ *disable:' | wc -l) = 0 && continue; echo "$diff" | grep -E '^\+ *(NoSourceSince|Disabled):' && continue; appid=`echo $f | sed -E -n 's,^metadata/([^/][^/]*)(\.yml|/signatures/.*),\1,p'`; export CHANGED="$CHANGED $appid"; done; - test -n "$(printf "$CHANGED" | tr -d '[:space:]')" || { echo "no packages need processing, exiting"; exit 0; } - test -d fdroidserver || mkdir fdroidserver - git ls-remote https://gitlab.com/fdroid/fdroidserver.git master - curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz | tar -xz --directory=fdroidserver --strip-components=1 - adduser --disabled-password --gecos "" vagrant - test -d /etc/sudoers.d || mkdir /etc/sudoers.d - echo "vagrant ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/vagrant - export VAGRANT_HOME=/home/vagrant - ln -s $CI_PROJECT_DIR/fdroidserver $VAGRANT_HOME/fdroidserver - cp -r $HOME/.android $VAGRANT_HOME/.android - for d in build logs tmp unsigned $ANDROID_HOME $VAGRANT_HOME/.android; do test -d $d || mkdir $d; chown -R vagrant $d; done - ln -s $ANDROID_HOME $VAGRANT_HOME/android-sdk - export ANDROID_HOME=$VAGRANT_HOME/android-sdk - bash fdroidserver/buildserver/setup-env-vars $ANDROID_HOME - bash fdroidserver/buildserver/provision-gradle - bash fdroidserver/buildserver/provision-apt-get-install http://deb.debian.org/debian - source /etc/profile.d/bsenv.sh - apt-get dist-upgrade # install fdroidserver from git, with deps from Debian, until fdroidserver # is stable enough to include all the things needed here - apt-get install -t stretch-backports fdroidserver python3-asn1crypto python3-ruamel.yaml yamllint - apt-get purge fdroidserver - export GRADLE_USER_HOME=$VAGRANT_HOME/.gradle # each `fdroid build --on-server` run expects sudo, then uninstalls it - export fdroid="sudo --preserve-env --user vagrant env PATH=$VAGRANT_HOME/fdroidserver:$PATH env PYTHONPATH=$VAGRANT_HOME/fdroidserver:$VAGRANT_HOME/fdroidserver/examples env PYTHONUNBUFFERED=true env TERM=$TERM env HOME=$VAGRANT_HOME fdroid" # use fdroidserver test keystore as placeholder since `fdroid publish` requires it - printf 'repo_keyalias\x3a sova\n' >> config.yml - printf 'keystorepass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml - printf 'keypass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml - printf 'keydname\x3a "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"\n' >> config.yml - keystore=$CI_PROJECT_DIR/fdroidserver/tests/keystore.jks - printf "keystore\x3a $keystore\n" >> config.yml - chown vagrant $keystore - for build in `./tools/find-changed-builds.py`; do set -x; apt-get install sudo; ./tools/setup-ndk-per-build.py $build $VAGRANT_HOME/android-ndk; chown -R vagrant $VAGRANT_HOME; $fdroid fetchsrclibs $build --verbose; $fdroid build --verbose --test --scan-binary --on-server --no-tarball $build; ./tools/build-contains-signatures.py $build || continue; apt-get install sudo; $fdroid publish --verbose $build; done # issuebot needs secrets to run, so it has to run under the 'fdroid' # group, therefore needs the trigger without secrets, there would be # no support for virustotal, github downloads, exodus privacy checks, # etc. That means it has to be triggered from the lint: job, which # runs in the fork's CI. The trigger-issuebot job adds the env var # FROM_CI_MERGE_REQUEST_IID which is required to have this job be # triggered. # # This job has to be called 'pages' because it deploys the JSON API # to GitLab Pages. pages: image: registry.gitlab.com/fdroid/ci-images-client:latest only: refs: - branches variables: - $FROM_CI_MERGE_REQUEST_IID artifacts: name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_JOB_ID}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}" paths: - metadata/ - public/ - repo/index-v1.json - repo/index.xml - tmp/apkcache.json - unsigned/ when: always # needs lots of git history since it has to compare the merge request to current master variables: GIT_DEPTH: "5000" script: - apt-get update - apt-get dist-upgrade # install fdroidserver from git, with deps from Debian, until fdroidserver # is stable enough to include all the things needed here - apt-get install -t stretch-backports fdroidserver python3-asn1crypto python3-ruamel.yaml python3-venv - apt-get purge fdroidserver - test -d fdroidserver || mkdir fdroidserver - git ls-remote https://gitlab.com/fdroid/fdroidserver.git master - curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz | tar -xz --directory=$CI_PROJECT_DIR/fdroidserver --strip-components=1 - export PATH="$CI_PROJECT_DIR/fdroidserver:$PATH" - export PYTHONPATH="$CI_PROJECT_DIR/fdroidserver:$CI_PROJECT_DIR/fdroidserver/examples" - export PYTHONUNBUFFERED=true - export GRADLE_USER_HOME=$PWD/.gradle - rm -rf $GRADLE_USER_HOME/fdroid - mkdir -p $GRADLE_USER_HOME/fdroid - git ls-remote https://gitlab.com/fdroid/gradle-plugins.git master - curl --silent https://gitlab.com/fdroid/gradle-plugins/-/archive/master/gradle-plugins-master.tar.gz | tar -xz --directory=$GRADLE_USER_HOME/fdroid --strip-components=1 - git ls-remote https://gitlab.com/fdroid/issuebot.git master - curl --silent https://gitlab.com/fdroid/issuebot/-/archive/master/issuebot-master.tar.gz | tar -xz --strip-components=1 - pyvenv --system-site-packages --clear issuebot-env - . issuebot-env/bin/activate - pip3 install wheel # get rid of the bdist_wheel error messages - pip3 install python-gitlab pygithub - ./issuebot.py # git_stats used to run here, redirect to new location - echo '' > public/index.html check_git_repos: image: debian:buster-slim stage: test only: refs: - schedules variables: - $CHECK_GIT_REPO == "true" artifacts: when: on_failure expire_in: 1 month paths: - public script: - apt-get update - apt-get -qy install --no-install-recommends ca-certificates git python3-colorama python3-yaml - tools/check-git-repo-availability.py || export EXITVALUE=1 - test -d public || mkdir public - cp `git status | grep -Eo 'metadata/.*\.yml'` public/ || true - exit $EXITVALUE