diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18503b47b1..a302102d66 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -112,6 +112,23 @@ checkupdates: git --no-pager diff --color=always --exit-code fi +git redirect: + stage: test + needs: [] + image: debian:bullseye-slim + rules: *app_verification_rules + script: + - apt-get -qy update + - apt-get -qy dist-upgrade + - apt-get -qy install --no-install-recommends ca-certificates git python3 python3-yaml + - *get_changed_apps + - test -n "$CHANGED" || exit 0 + - tools/rewrite-git-redirects.py $CHANGED + - git --no-pager diff --color=always --exit-code || { + printf "\x1b[31mERROR git URLs should be direct, not redirects!\x1b[0m\n"; + exit 1; + } + fdroid lint: stage: test needs: [] diff --git a/tools/rewrite-git-redirects.py b/tools/rewrite-git-redirects.py index e67f56a38e..de0d06d73b 100755 --- a/tools/rewrite-git-redirects.py +++ b/tools/rewrite-git-redirects.py @@ -1,20 +1,42 @@ #!/usr/bin/env python3 # -# -# GitLab gives a warning every time if the git URL is redirected. So this -# rewrites all GitLab URLs so they are no longer a redirect. +# GitLab gives a warning every time if the git URL is redirected. So +# this rewrites all GitLab URLs so they are no longer a redirect. +# This also allows the gitlab.com URLs to be used with Tor, and gives +# other benefits, like a simplier security attack surface. import glob import os import re +import subprocess import sys - import yaml + +def is_git_redirect(url): + """Check if git is served a redirect""" + host = url.split('/')[2] + p = subprocess.run( + [ + 'git', + '-c', + 'url.https://git:nopw@{host}.insteadOf=https://{host}'.format(host=host), + '-c', + 'http.followRedirects=false', + 'ls-remote', + '--heads', + url, + 'main', + ], + capture_output=True, + ) + return p.returncode != 0 + + os.chdir(os.path.dirname(__file__) + '/../') if len(sys.argv) > 1: - files = sys.argv[1:] + files = ['metadata/%s.yml' % f for f in sys.argv[1:]] else: files = sorted(glob.glob('metadata/*.yml')) @@ -22,15 +44,27 @@ pattern = re.compile(r'Repo: .*') for f in files: with open(f) as fp: data = yaml.safe_load(fp) - repo_url = None - if 'Repo' in data: - repo_url = data['Repo'].strip().rstrip('/') + repo_url = data.get('Repo', '').strip() if ( - repo_url - and not repo_url.endswith('.git') - and repo_url.startswith('https://gitlab') + 'Repo' not in data + or data.get('RepoType') != 'git' + or data.get('ArchivePolicy') == '0 versions' ): - new_url = repo_url + '.git' + continue + + if not is_git_redirect(repo_url): + continue + + new_url = None + for url in [ + repo_url.rstrip('/') + '.git', + repo_url.rstrip('/'), + ]: + if not is_git_redirect(url): + new_url = url + break + + if new_url: print("Repo:", data['Repo'], "\n --> " + new_url + "'") with open(f) as fp: raw = fp.read()