Merge branch 'srclibs-cleanup' into 'master'

srclibs cleanup

See merge request fdroid/fdroiddata!6895

.gitlab-ci.yml

@@ -21,17 +21,17 @@ lint:
     - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
         git fetch https://gitlab.com/fdroid/fdroiddata;
         test -d build || mkdir build;
-        set -x;
-        for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
+        files=`git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`;
+        for f in $files; do
            appid=`echo $f | sed -n -e 's,^metadata/\([^/][^/]*\)\.yml,\1,p'`;
            if [ -n "$appid" ]; then
              export CHANGED="$CHANGED $appid";
              testcmd="git -C build clone `sed -En 's,^Repo\W +(https?://),\1u:p@,p' $f`";
-           else
-             continue;
            fi;
-           grep -q '^RepoType\W *git$' $f && tail -1 $f | grep -qv '^NoSourceSince:' && $testcmd;
         done;
+        set -x;
+        apt-get install python3-colorama;
+        ./tools/check-git-repo-availability.py $files;
         ./tools/audit-gradle.py $CHANGED;
         set +x;
       fi
@@ -117,7 +117,7 @@ check_git_repos:
       - public
   script:
     - apt-get update
-    - apt-get -qy install --no-install-recommends ca-certificates git python3-yaml
+    - apt-get -qy install --no-install-recommends ca-certificates git python3-colorama python3-yaml
     - tools/check-git-repo-availability.py || export EXITVALUE=1
     - test -d public || mkdir public
     - cp `git status | grep -Eo 'metadata/.*\.yml'` public/ || true

srclibs/BingTranslator.yml

@@ -1,12 +1,2 @@
-# Source details (the only mandatory fields)
 RepoType: git
 Repo: https://github.com/boatmeme/microsoft-translator-java-api.git
-
-# Comma-separated list of subdirs to use. The first existing subdirectory
-# found between those given will be used. If none is found or provided, the
-# root of the repo directory will be used instead.
-Subdir: 
-
-# Any extra commands to prepare the source library
-Prepare: |
-  

srclibs/BouncyCastle.yml

@@ -1,12 +1,2 @@
-# Source details (the only mandatory fields)
 RepoType: git
 Repo: https://github.com/bcgit/bc-java.git
-
-# Comma-separated list of subdirs to use. The first existing subdirectory
-# found between those given will be used. If none is found or provided, the
-# root of the repo directory will be used instead.
-Subdir: 
-
-# Any extra commands to prepare the source library
-Prepare: |
-  

srclibs/FluffyLocations.yml

@@ -1,12 +1,3 @@
-# Source details (the only mandatory fields)
 RepoType: svn
 Repo: http://little-fluffy-location-library.googlecode.com/svn/trunk/
-
-# Comma-separated list of subdirs to use. The first existing subdirectory
-# found between those given will be used. If none is found or provided, the
-# root of the repo directory will be used instead.
 Subdir: little-fluffy-location-library/LittleFluffyLocationLibrary
-
-# Any extra commands to prepare the source library
-Prepare: |
-  

srclibs/JSoup.yml

@@ -1,12 +1,2 @@
-# Source details (the only mandatory fields)
 RepoType: git
 Repo: https://github.com/jhy/jsoup.git
-
-# Comma-separated list of subdirs to use. The first existing subdirectory
-# found between those given will be used. If none is found or provided, the
-# root of the repo directory will be used instead.
-Subdir: 
-
-# Any extra commands to prepare the source library
-Prepare: |
-  

srclibs/MSQLite.yml

@@ -1,12 +1,2 @@
-# Source details (the only mandatory fields)
 RepoType: git
 Repo: https://github.com/mick88/MSQLite.git
-
-# Comma-separated list of subdirs to use. The first existing subdirectory
-# found between those given will be used. If none is found or provided, the
-# root of the repo directory will be used instead.
-Subdir: 
-
-# Any extra commands to prepare the source library
-Prepare: |
-  

srclibs/SystemBarTint.yml

@@ -1,12 +1,3 @@
-# Source details (the only mandatory fields)
 RepoType: git
 Repo: https://github.com/jgilfelt/SystemBarTint
-
-# Comma-separated list of subdirs to use. The first existing subdirectory
-# found between those given will be used. If none is found or provided, the
-# root of the repo directory will be used instead.
 Subdir: library
-
-# Any extra commands to prepare the source library
-Prepare: |
-  

tools/check-git-repo-availability.py

@@ -1,28 +1,47 @@
 #!/usr/bin/env python3
 
 import glob
+import os
 import re
 import subprocess
 import sys
 import yaml
+from colorama import Fore, Style
+try:
+    from yaml import CSafeLoader as SafeLoader
+except ImportError:
+    from yaml import SafeLoader
 
 if len(sys.argv) > 1:
     files = sys.argv[1:]
 else:
-    files = sorted(glob.glob('metadata/*.yml'))
+    files = sorted(glob.glob('metadata/*.yml') + glob.glob('srclibs/*.yml'))
 
 errors = dict()
 for f in files:
+    f = os.path.relpath(f)
+    if not f.startswith('metadata/') and not f.startswith('srclibs/'):
+        continue
     if not f.endswith('.yml'):
         print('\n' + f + ':\nThis only runs on YAML files (.yml), ignoring.')
         continue
     with open(f) as fp:
-        data = yaml.load(fp)
+        data = yaml.load(fp, Loader=SafeLoader)
 
-    url = data.get('Repo')
-    if not url or 'NoSourceSince' in data.keys():
+    if not data:
+        msg = 'ERROR: %s: empty file!' % f
+        print(Fore.RED + msg + Style.RESET_ALL)
+        errors[f] = msg
         continue
-    if data['RepoType'] != 'git':
+    url = data.get('Repo')
+    if not url:
+        msg = 'ERROR: %s: no Repo: set!' % f
+        print(Fore.RED + msg + Style.RESET_ALL)
+        errors[f] = msg
+        continue
+    if 'NoSourceSince' in data.keys():
+        continue
+    if data.get('RepoType') != 'git':
         continue
 
     # from class vcs_git() in fdroidserver/common.py
@@ -47,10 +66,18 @@ for f in files:
 
     p = subprocess.run(['git', ] + git_config + ['ls-remote', '--exit-code', '-h', url],
                        env=env,
-                       capture_output=True)
+                       stdout=subprocess.DEVNULL,
+                       stderr=subprocess.PIPE)
     if p.returncode != 0:
         with open(f) as fp:
             raw = fp.read()
+        print('\n' + f + ':')
+        print('%s%s%s' % (Fore.RED, p.stderr.decode(), Style.RESET_ALL))
+        errors[f] = p.stderr
+
+        if not f.startswith('metadata/'):
+            continue
+        # this rewriting only works with metadata files, not srclibs
         with open(f, 'w') as fp:
             fp.write(re.sub(r'(Repo|RepoType):.*\n{1,2}', r'', raw))
             builds = data.get('Builds')
@@ -59,16 +86,12 @@ for f in files:
                 # if YAML will think its a float, quote it
                 try:
                     float(versionName)
-                    fp.write("\nNoSourceSince: '" + versionName + "'")
+                    fp.write("\nNoSourceSince: '%s'" % versionName)
                 except ValueError:
-                    fp.write("\nNoSourceSince: " + versionName)
+                    fp.write("\nNoSourceSince: %s" % versionName)
                 fp.write('\n')
 
-        print('\n' + f + ':')
-        print(p.stderr.decode())
-        errors[f] = p.stderr
-
 errorcount = len(errors)
 if errorcount > 0:
-    print('\nFound', errorcount, 'errors.')
+    print(Fore.RED + '\nFound', errorcount, 'errors.' + Style.RESET_ALL)
 sys.exit(errorcount)