Add subscription repo upload script

icinga-build-upload-aptly-subscription

+#!/usr/bin/env python
+
+import sys
+import os
+import os.path
+import argparse
+import fnmatch
+from hashlib import sha1
+import requests
+import json
+import re
+
+# internals
+APTLY_SESSION = None
+
+CI_JOB_NAME = 'CI_JOB_NAME'
+CI_JOB_UPLOAD_PREFIX = 'upload[:/]'
+ICINGA_BUILD_TYPE = 'ICINGA_BUILD_TYPE'
+ICINGA_BUILD_RELEASE_TYPE = 'ICINGA_BUILD_RELEASE_TYPE'
+ICINGA_BUILD_TYPE_DEFAULT = 'release'
+
+UPLOAD_TYPE_DEB = 'DEB'
+UPLOAD_TYPE_RPM = 'RPM'
+
+def scan_dir(path, pattern):
+    found = []
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            if fnmatch.fnmatch(file, pattern):
+                found.append(os.path.join(root, file))
+    return found
+
+def detect_rpm(path):
+    source = scan_dir(path, '*.src.rpm')
+    rpms = scan_dir(path, '*.rpm')
+    if len(source) == 0:
+        return []
+    elif len(source) != 1:
+        raise StandardError, 'There more than one source RPM in ' + path
+
+    if len(rpms) < 2:
+        raise StandardError, 'There should be at least 2 RPMS in ' + path
+
+    return rpms
+
+def detect_deb(path):
+    source = scan_dir(path, '*.dsc')
+    tarballs = scan_dir(path, '*.tar*')
+    changes = scan_dir(path, '*.changes')
+    debs = scan_dir(path, '*.deb')
+
+    if len(source) == 0:
+        return []
+    elif len(source) != 1:
+        raise StandardError, 'There more than one source DSC in ' + path
+
+    files = source + tarballs + changes + debs
+    if len(files) < 2:
+        raise StandardError, 'There should be at least 2 files in ' + path
+
+    return files
+
+def sha1_file(path):
+    h = sha1()
+    with open(path, 'rb') as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+def upload_checksum(files):
+    files = sorted(files)
+    h = sha1()
+    d = dict()
+    for file in files:
+        name = os.path.basename(file)
+        d[name] = fh = sha1_file(file)
+        h.update(fh)
+    return (h.hexdigest(), d)
+
+def ci_split_name():
+    """
+    Split the CI_JOB_NAME into target and release
+    e.g. upload/debian/stretch -> ['debian', 'stretch']
+    or upload/stack/dev/debian/stretch -> ['stack/dev/debian', 'stretch']
+    """
+    job_name = os.environ.get(CI_JOB_NAME)
+    if job_name:
+        match = re.match("^%s(.+)/([^/]+)$" % CI_JOB_UPLOAD_PREFIX, job_name)
+        if match:
+            return match.groups()
+
+    return None
+
+def get_release_type():
+    """
+    Get ICINGA_BUILD_RELEASE_TYPE or ICINGA_BUILD_TYPE from environ
+    """
+    if os.environ.has_key(ICINGA_BUILD_RELEASE_TYPE):
+        return os.environ.get(ICINGA_BUILD_RELEASE_TYPE)
+
+    if os.environ.has_key(ICINGA_BUILD_TYPE):
+        return os.environ.get(ICINGA_BUILD_TYPE)
+
+    return ICINGA_BUILD_TYPE_DEFAULT
+
+def ci_release(upload_type, release):
+    """
+    Build the release name from CI_JOB_NAME and ICINGA_BUILD_TYPE
+
+    Examples:
+        DEB / stretch / release -> icinga-stretch
+        DEB / stretch / snapshot -> icinga-stretch-snapshots
+        DEB / stretch / Y -> icinga-stretch-Y
+
+        RPM / 7 / release -> 7/release
+        RPM / X / Y -> X/Y
+    """
+    build_type = get_release_type()
+
+    if upload_type == UPLOAD_TYPE_DEB:
+        publish_release = 'icinga-' + release
+        if build_type != 'release':
+            publish_release += '-' + build_type
+
+        if build_type == 'snapshot':
+            publish_release += 's' # snapshots
+
+        return publish_release
+    elif upload_type == UPLOAD_TYPE_RPM:
+        return release + '/' + build_type
+    else:
+        raise StandardError, "Unknown upload type %s" % upload_type
+
+def ci_repo(upload_type, target, release):
+    """
+    Build the aptly repo from target, release and ICINGA_BUILD_TYPE (only for DEB)
+
+    Examples:
+        icinga-debian-stretch-release
+        icinga-debian-stretch-snapshot
+    """
+    if upload_type != UPLOAD_TYPE_DEB:
+        raise StandardError, "Repo can only be set on DEB uploads!"
+
+    build_type = get_release_type()
+
+    return 'icinga-%s-%s-%s' % (target, release, build_type)
+
+def aptly_session():
+    global APTLY_SESSION
+    if APTLY_SESSION is None:
+        APTLY_SESSION = s = requests.Session()
+        if args.username and args.password:
+            s.auth = (args.username, args.password)
+        if args.insecure:
+            s.verify = False
+
+    return APTLY_SESSION
+
+def aptly_url(url):
+    return args.server + url
+
+parser = argparse.ArgumentParser(description='Uploading build results to an Aptly server')
+parser.add_argument('--server', help='APTLY API service to talk to (e.g. http://127.0.0.1:8080/api)',
+                    default=os.environ.get('APTLY_SERVER', 'http://127.0.0.1:8080/api'), metavar='APTLY_SERVER')
+parser.add_argument('--username', help='APTLY API username',
+                    default=os.environ.get('APTLY_USERNAME'), metavar='APTLY_USERNAME')
+parser.add_argument('--password', help='APTLY API password',
+                    default=os.environ.get('APTLY_PASSWORD'), metavar='APTLY_PASSWORD')
+parser.add_argument('--result', metavar='path',
+                    default='build/', help='Build result to upload')
+parser.add_argument('--target', metavar='target',
+                    help='Repository to install the package to (e.g. stack/dev/epel or stack/dev/ubuntu)')
+parser.add_argument('--release', metavar='release',
+                    help='Version of the repository to install to (e.g. 7 or icinga-xenial)')
+parser.add_argument('--repo', metavar='repo',
+                    help='Specific repository name in aptly')
+parser.add_argument('--architectures', metavar='list',
+                    default=os.environ.get('ICINGA_BUILD_DEB_DEFAULT_ARCH', 'amd64'),
+                    help=('Specify list of architectures to publish the repo with,'
+                          'separated by comma (e.g. amd64,i386 or armhf)'))
+parser.add_argument('--insecure', action='store_true', help='Disable SSL verification')
+parser.add_argument('--noop', action='store_true', help='Only prepare upload')
+
+args = parser.parse_args()
+
+if not args.server and not args.noop:
+    raise StandardError, "Specifying an aptly server is required (--server or APTLY_SERVER)"
+
+if not os.path.exists(args.result):
+    raise StandardError, "Result path '%s' does not exist!" % (args.result)
+
+rpms = detect_rpm(args.result)
+debs = detect_deb(args.result)
+files = None
+if rpms:
+    type = UPLOAD_TYPE_RPM
+    files = rpms
+elif debs:
+    type = UPLOAD_TYPE_DEB
+    files = debs
+
+if not files:
+    raise StandardError, 'No packages found in %s' % (args.result)
+
+pair = ci_split_name()
+
+if not args.target:
+    if pair:
+        args.target = pair[0]
+    else:
+        raise StandardError, "Could not detect --target from %s, please specify!" % CI_JOB_NAME
+
+if not args.release:
+    if pair:
+        args.release = ci_release(type, pair[1])
+    else:
+        raise StandardError, "Could not detect --release from %s, please specify!" % CI_JOB_NAME
+
+if not args.repo and type == UPLOAD_TYPE_DEB:
+    if pair:
+        args.repo = ci_repo(type, *pair)
+    else:
+        raise StandardError, "Could not detect --repo from %s, please specify!" % CI_JOB_NAME
+
+(checksum, checksums) = upload_checksum(files)
+upload_prefix = re.sub('[/\-_]+', '_', args.target + '_' + args.release)
+upload_name = '%s_%s' % (upload_prefix, checksum)
+
+# meta and upload file
+upload_meta = {
+    'target': args.target,
+    'release': args.release,
+    'type': type,
+    'checksums': checksums,
+}
+
+if args.repo and type == UPLOAD_TYPE_DEB:
+    upload_meta['repo'] = args.repo
+
+if args.architectures:
+    upload_meta['architectures'] = re.split(r'\s*,\s*', args.architectures)
+
+    # always add i386 if amd64 is base arch
+    if 'amd64' in upload_meta['architectures'] and not 'i386' in upload_meta['architectures']:
+        upload_meta['architectures'].append('i386')
+
+print "Prepared upload to: %s" % (aptly_url('/files/' + upload_name))
+print "Metadata is:" + json.dumps(upload_meta, sort_keys=True, indent=4)
+print
+
+if args.noop:
+    print "Running in noop mode, stopping here!"
+    sys.exit(0)
+
+# ensure target is absent
+r = aptly_session().delete(aptly_url('/files/' + upload_name))
+if r.status_code == requests.codes.ok:
+    print "Deleted existing upload %s" % (upload_name)
+elif r.status_code != requests.codes.not_found:
+    raise StandardError, 'Unexpected result code: %s' % (r.status_code)
+
+# uploading files
+upload_url = aptly_url('/files/subscription/' + upload_name)
+print "Uploading %d files to %s" % (len(files), upload_name)
+for file in files:
+    file_data = [('file', (file, open(file, 'rb')))]
+    r = aptly_session().post(upload_url, files=file_data)
+    if r.status_code == requests.codes.ok:
+        print "Upload successful: %s" % (file)
+    else:
+        raise StandardError, "Upload failed for %s - http status: %s - message:\n%s" % (upload_name, r.status_code, r.text[:30])
+
+file_data = [('file', ('upload.json', json.dumps(upload_meta)))]
+r = aptly_session().post(upload_url, files=file_data)
+if r.status_code == requests.codes.ok:
+    print 'Metadata upload successful.'
+else:
+    raise StandardError, "Upload metadata failed for %s - http status: %s - message:\n%s" % (upload_name, r.status_code, r.text[:30])