diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fcd0ad92e2..fc651c27f1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,17 +21,17 @@ lint: - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then git fetch https://gitlab.com/fdroid/fdroiddata; test -d build || mkdir build; - set -x; - for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do + files=`git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; + for f in $files; do appid=`echo $f | sed -n -e 's,^metadata/\([^/][^/]*\)\.yml,\1,p'`; if [ -n "$appid" ]; then export CHANGED="$CHANGED $appid"; testcmd="git -C build clone `sed -En 's,^Repo\W +(https?://),\1u:p@,p' $f`"; - else - continue; fi; - grep -q '^RepoType\W *git$' $f && tail -1 $f | grep -qv '^NoSourceSince:' && $testcmd; done; + set -x; + apt-get install python3-colorama; + ./tools/check-git-repo-availability.py $files; ./tools/audit-gradle.py $CHANGED; set +x; fi @@ -117,7 +117,7 @@ check_git_repos: - public script: - apt-get update - - apt-get -qy install --no-install-recommends ca-certificates git python3-yaml + - apt-get -qy install --no-install-recommends ca-certificates git python3-colorama python3-yaml - tools/check-git-repo-availability.py || export EXITVALUE=1 - test -d public || mkdir public - cp `git status | grep -Eo 'metadata/.*\.yml'` public/ || true diff --git a/srclibs/BingTranslator.yml b/srclibs/BingTranslator.yml index 15be24c639..96bdd25069 100644 --- a/srclibs/BingTranslator.yml +++ b/srclibs/BingTranslator.yml @@ -1,12 +1,2 @@ -# Source details (the only mandatory fields) RepoType: git Repo: https://github.com/boatmeme/microsoft-translator-java-api.git - -# Comma-separated list of subdirs to use. The first existing subdirectory -# found between those given will be used. If none is found or provided, the -# root of the repo directory will be used instead. -Subdir: - -# Any extra commands to prepare the source library -Prepare: | - diff --git a/srclibs/BouncyCastle.yml b/srclibs/BouncyCastle.yml index 2b3374a9d2..4ac717ce27 100644 --- a/srclibs/BouncyCastle.yml +++ b/srclibs/BouncyCastle.yml @@ -1,12 +1,2 @@ -# Source details (the only mandatory fields) RepoType: git Repo: https://github.com/bcgit/bc-java.git - -# Comma-separated list of subdirs to use. The first existing subdirectory -# found between those given will be used. If none is found or provided, the -# root of the repo directory will be used instead. -Subdir: - -# Any extra commands to prepare the source library -Prepare: | - diff --git a/srclibs/FluffyLocations.yml b/srclibs/FluffyLocations.yml index 2eb4ee8aa7..b8f488f847 100644 --- a/srclibs/FluffyLocations.yml +++ b/srclibs/FluffyLocations.yml @@ -1,12 +1,3 @@ -# Source details (the only mandatory fields) RepoType: svn Repo: http://little-fluffy-location-library.googlecode.com/svn/trunk/ - -# Comma-separated list of subdirs to use. The first existing subdirectory -# found between those given will be used. If none is found or provided, the -# root of the repo directory will be used instead. Subdir: little-fluffy-location-library/LittleFluffyLocationLibrary - -# Any extra commands to prepare the source library -Prepare: | - diff --git a/srclibs/JSoup.yml b/srclibs/JSoup.yml index 718cc25430..be8b3212f1 100644 --- a/srclibs/JSoup.yml +++ b/srclibs/JSoup.yml @@ -1,12 +1,2 @@ -# Source details (the only mandatory fields) RepoType: git Repo: https://github.com/jhy/jsoup.git - -# Comma-separated list of subdirs to use. The first existing subdirectory -# found between those given will be used. If none is found or provided, the -# root of the repo directory will be used instead. -Subdir: - -# Any extra commands to prepare the source library -Prepare: | - diff --git a/srclibs/MSQLite.yml b/srclibs/MSQLite.yml index fb16c02e3f..75eb973471 100644 --- a/srclibs/MSQLite.yml +++ b/srclibs/MSQLite.yml @@ -1,12 +1,2 @@ -# Source details (the only mandatory fields) RepoType: git Repo: https://github.com/mick88/MSQLite.git - -# Comma-separated list of subdirs to use. The first existing subdirectory -# found between those given will be used. If none is found or provided, the -# root of the repo directory will be used instead. -Subdir: - -# Any extra commands to prepare the source library -Prepare: | - diff --git a/srclibs/SystemBarTint.yml b/srclibs/SystemBarTint.yml index bcb6fdec9c..db106cbb34 100644 --- a/srclibs/SystemBarTint.yml +++ b/srclibs/SystemBarTint.yml @@ -1,12 +1,3 @@ -# Source details (the only mandatory fields) RepoType: git Repo: https://github.com/jgilfelt/SystemBarTint - -# Comma-separated list of subdirs to use. The first existing subdirectory -# found between those given will be used. If none is found or provided, the -# root of the repo directory will be used instead. Subdir: library - -# Any extra commands to prepare the source library -Prepare: | - diff --git a/tools/check-git-repo-availability.py b/tools/check-git-repo-availability.py index bdb8b67767..8239b9baef 100755 --- a/tools/check-git-repo-availability.py +++ b/tools/check-git-repo-availability.py @@ -1,28 +1,47 @@ #!/usr/bin/env python3 import glob +import os import re import subprocess import sys import yaml +from colorama import Fore, Style +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader if len(sys.argv) > 1: files = sys.argv[1:] else: - files = sorted(glob.glob('metadata/*.yml')) + files = sorted(glob.glob('metadata/*.yml') + glob.glob('srclibs/*.yml')) errors = dict() for f in files: + f = os.path.relpath(f) + if not f.startswith('metadata/') and not f.startswith('srclibs/'): + continue if not f.endswith('.yml'): print('\n' + f + ':\nThis only runs on YAML files (.yml), ignoring.') continue with open(f) as fp: - data = yaml.load(fp) + data = yaml.load(fp, Loader=SafeLoader) - url = data.get('Repo') - if not url or 'NoSourceSince' in data.keys(): + if not data: + msg = 'ERROR: %s: empty file!' % f + print(Fore.RED + msg + Style.RESET_ALL) + errors[f] = msg continue - if data['RepoType'] != 'git': + url = data.get('Repo') + if not url: + msg = 'ERROR: %s: no Repo: set!' % f + print(Fore.RED + msg + Style.RESET_ALL) + errors[f] = msg + continue + if 'NoSourceSince' in data.keys(): + continue + if data.get('RepoType') != 'git': continue # from class vcs_git() in fdroidserver/common.py @@ -47,10 +66,18 @@ for f in files: p = subprocess.run(['git', ] + git_config + ['ls-remote', '--exit-code', '-h', url], env=env, - capture_output=True) + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE) if p.returncode != 0: with open(f) as fp: raw = fp.read() + print('\n' + f + ':') + print('%s%s%s' % (Fore.RED, p.stderr.decode(), Style.RESET_ALL)) + errors[f] = p.stderr + + if not f.startswith('metadata/'): + continue + # this rewriting only works with metadata files, not srclibs with open(f, 'w') as fp: fp.write(re.sub(r'(Repo|RepoType):.*\n{1,2}', r'', raw)) builds = data.get('Builds') @@ -59,16 +86,12 @@ for f in files: # if YAML will think its a float, quote it try: float(versionName) - fp.write("\nNoSourceSince: '" + versionName + "'") + fp.write("\nNoSourceSince: '%s'" % versionName) except ValueError: - fp.write("\nNoSourceSince: " + versionName) + fp.write("\nNoSourceSince: %s" % versionName) fp.write('\n') - print('\n' + f + ':') - print(p.stderr.decode()) - errors[f] = p.stderr - errorcount = len(errors) if errorcount > 0: - print('\nFound', errorcount, 'errors.') + print(Fore.RED + '\nFound', errorcount, 'errors.' + Style.RESET_ALL) sys.exit(errorcount)