From 1ff1fa3aaea91f076ec8e55af39346a770776831 Mon Sep 17 00:00:00 2001
From: Henrik Triem <henrik.triem@netways.de>
Date: Mon, 8 Nov 2021 16:26:58 +0100
Subject: [PATCH] Add subscription repo upload script
---
icinga-build-upload-aptly-subscription | 279 +++++++++++++++++++++++++
1 file changed, 279 insertions(+)
create mode 100755 icinga-build-upload-aptly-subscription
diff --git a/icinga-build-upload-aptly-subscription b/icinga-build-upload-aptly-subscription
new file mode 100755
index 0000000..12d3beb
--- /dev/null
+++ b/icinga-build-upload-aptly-subscription
@@ -0,0 +1,279 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import os.path
+import argparse
+import fnmatch
+from hashlib import sha1
+import requests
+import json
+import re
+
+# internals
+APTLY_SESSION = None
+
+CI_JOB_NAME = 'CI_JOB_NAME'
+CI_JOB_UPLOAD_PREFIX = 'upload[:/]'
+ICINGA_BUILD_TYPE = 'ICINGA_BUILD_TYPE'
+ICINGA_BUILD_RELEASE_TYPE = 'ICINGA_BUILD_RELEASE_TYPE'
+ICINGA_BUILD_TYPE_DEFAULT = 'release'
+
+UPLOAD_TYPE_DEB = 'DEB'
+UPLOAD_TYPE_RPM = 'RPM'
+
+def scan_dir(path, pattern):
+ found = []
+ for root, dirs, files in os.walk(path):
+ for file in files:
+ if fnmatch.fnmatch(file, pattern):
+ found.append(os.path.join(root, file))
+ return found
+
+def detect_rpm(path):
+ source = scan_dir(path, '*.src.rpm')
+ rpms = scan_dir(path, '*.rpm')
+ if len(source) == 0:
+ return []
+ elif len(source) != 1:
+ raise StandardError, 'There more than one source RPM in ' + path
+
+ if len(rpms) < 2:
+ raise StandardError, 'There should be at least 2 RPMS in ' + path
+
+ return rpms
+
+def detect_deb(path):
+ source = scan_dir(path, '*.dsc')
+ tarballs = scan_dir(path, '*.tar*')
+ changes = scan_dir(path, '*.changes')
+ debs = scan_dir(path, '*.deb')
+
+ if len(source) == 0:
+ return []
+ elif len(source) != 1:
+ raise StandardError, 'There more than one source DSC in ' + path
+
+ files = source + tarballs + changes + debs
+ if len(files) < 2:
+ raise StandardError, 'There should be at least 2 files in ' + path
+
+ return files
+
+def sha1_file(path):
+ h = sha1()
+ with open(path, 'rb') as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ h.update(chunk)
+ return h.hexdigest()
+
+def upload_checksum(files):
+ files = sorted(files)
+ h = sha1()
+ d = dict()
+ for file in files:
+ name = os.path.basename(file)
+ d[name] = fh = sha1_file(file)
+ h.update(fh)
+ return (h.hexdigest(), d)
+
+def ci_split_name():
+ """
+ Split the CI_JOB_NAME into target and release
+ e.g. upload/debian/stretch -> ['debian', 'stretch']
+ or upload/stack/dev/debian/stretch -> ['stack/dev/debian', 'stretch']
+ """
+ job_name = os.environ.get(CI_JOB_NAME)
+ if job_name:
+ match = re.match("^%s(.+)/([^/]+)$" % CI_JOB_UPLOAD_PREFIX, job_name)
+ if match:
+ return match.groups()
+
+ return None
+
+def get_release_type():
+ """
+ Get ICINGA_BUILD_RELEASE_TYPE or ICINGA_BUILD_TYPE from environ
+ """
+ if os.environ.has_key(ICINGA_BUILD_RELEASE_TYPE):
+ return os.environ.get(ICINGA_BUILD_RELEASE_TYPE)
+
+ if os.environ.has_key(ICINGA_BUILD_TYPE):
+ return os.environ.get(ICINGA_BUILD_TYPE)
+
+ return ICINGA_BUILD_TYPE_DEFAULT
+
+def ci_release(upload_type, release):
+ """
+ Build the release name from CI_JOB_NAME and ICINGA_BUILD_TYPE
+
+ Examples:
+ DEB / stretch / release -> icinga-stretch
+ DEB / stretch / snapshot -> icinga-stretch-snapshots
+ DEB / stretch / Y -> icinga-stretch-Y
+
+ RPM / 7 / release -> 7/release
+ RPM / X / Y -> X/Y
+ """
+ build_type = get_release_type()
+
+ if upload_type == UPLOAD_TYPE_DEB:
+ publish_release = 'icinga-' + release
+ if build_type != 'release':
+ publish_release += '-' + build_type
+
+ if build_type == 'snapshot':
+ publish_release += 's' # snapshots
+
+ return publish_release
+ elif upload_type == UPLOAD_TYPE_RPM:
+ return release + '/' + build_type
+ else:
+ raise StandardError, "Unknown upload type %s" % upload_type
+
+def ci_repo(upload_type, target, release):
+ """
+ Build the aptly repo from target, release and ICINGA_BUILD_TYPE (only for DEB)
+
+ Examples:
+ icinga-debian-stretch-release
+ icinga-debian-stretch-snapshot
+ """
+ if upload_type != UPLOAD_TYPE_DEB:
+ raise StandardError, "Repo can only be set on DEB uploads!"
+
+ build_type = get_release_type()
+
+ return 'icinga-%s-%s-%s' % (target, release, build_type)
+
+def aptly_session():
+ global APTLY_SESSION
+ if APTLY_SESSION is None:
+ APTLY_SESSION = s = requests.Session()
+ if args.username and args.password:
+ s.auth = (args.username, args.password)
+ if args.insecure:
+ s.verify = False
+
+ return APTLY_SESSION
+
+def aptly_url(url):
+ return args.server + url
+
+parser = argparse.ArgumentParser(description='Uploading build results to an Aptly server')
+parser.add_argument('--server', help='APTLY API service to talk to (e.g. http://127.0.0.1:8080/api)',
+ default=os.environ.get('APTLY_SERVER', 'http://127.0.0.1:8080/api'), metavar='APTLY_SERVER')
+parser.add_argument('--username', help='APTLY API username',
+ default=os.environ.get('APTLY_USERNAME'), metavar='APTLY_USERNAME')
+parser.add_argument('--password', help='APTLY API password',
+ default=os.environ.get('APTLY_PASSWORD'), metavar='APTLY_PASSWORD')
+parser.add_argument('--result', metavar='path',
+ default='build/', help='Build result to upload')
+parser.add_argument('--target', metavar='target',
+ help='Repository to install the package to (e.g. stack/dev/epel or stack/dev/ubuntu)')
+parser.add_argument('--release', metavar='release',
+ help='Version of the repository to install to (e.g. 7 or icinga-xenial)')
+parser.add_argument('--repo', metavar='repo',
+ help='Specific repository name in aptly')
+parser.add_argument('--architectures', metavar='list',
+ default=os.environ.get('ICINGA_BUILD_DEB_DEFAULT_ARCH', 'amd64'),
+ help=('Specify list of architectures to publish the repo with,'
+ 'separated by comma (e.g. amd64,i386 or armhf)'))
+parser.add_argument('--insecure', action='store_true', help='Disable SSL verification')
+parser.add_argument('--noop', action='store_true', help='Only prepare upload')
+
+args = parser.parse_args()
+
+if not args.server and not args.noop:
+ raise StandardError, "Specifying an aptly server is required (--server or APTLY_SERVER)"
+
+if not os.path.exists(args.result):
+ raise StandardError, "Result path '%s' does not exist!" % (args.result)
+
+rpms = detect_rpm(args.result)
+debs = detect_deb(args.result)
+files = None
+if rpms:
+ type = UPLOAD_TYPE_RPM
+ files = rpms
+elif debs:
+ type = UPLOAD_TYPE_DEB
+ files = debs
+
+if not files:
+ raise StandardError, 'No packages found in %s' % (args.result)
+
+pair = ci_split_name()
+
+if not args.target:
+ if pair:
+ args.target = pair[0]
+ else:
+ raise StandardError, "Could not detect --target from %s, please specify!" % CI_JOB_NAME
+
+if not args.release:
+ if pair:
+ args.release = ci_release(type, pair[1])
+ else:
+ raise StandardError, "Could not detect --release from %s, please specify!" % CI_JOB_NAME
+
+if not args.repo and type == UPLOAD_TYPE_DEB:
+ if pair:
+ args.repo = ci_repo(type, *pair)
+ else:
+ raise StandardError, "Could not detect --repo from %s, please specify!" % CI_JOB_NAME
+
+(checksum, checksums) = upload_checksum(files)
+upload_prefix = re.sub('[/\-_]+', '_', args.target + '_' + args.release)
+upload_name = '%s_%s' % (upload_prefix, checksum)
+
+# meta and upload file
+upload_meta = {
+ 'target': args.target,
+ 'release': args.release,
+ 'type': type,
+ 'checksums': checksums,
+}
+
+if args.repo and type == UPLOAD_TYPE_DEB:
+ upload_meta['repo'] = args.repo
+
+if args.architectures:
+ upload_meta['architectures'] = re.split(r'\s*,\s*', args.architectures)
+
+ # always add i386 if amd64 is base arch
+ if 'amd64' in upload_meta['architectures'] and not 'i386' in upload_meta['architectures']:
+ upload_meta['architectures'].append('i386')
+
+print "Prepared upload to: %s" % (aptly_url('/files/' + upload_name))
+print "Metadata is:" + json.dumps(upload_meta, sort_keys=True, indent=4)
+print
+
+if args.noop:
+ print "Running in noop mode, stopping here!"
+ sys.exit(0)
+
+# ensure target is absent
+r = aptly_session().delete(aptly_url('/files/' + upload_name))
+if r.status_code == requests.codes.ok:
+ print "Deleted existing upload %s" % (upload_name)
+elif r.status_code != requests.codes.not_found:
+ raise StandardError, 'Unexpected result code: %s' % (r.status_code)
+
+# uploading files
+upload_url = aptly_url('/files/subscription/' + upload_name)
+print "Uploading %d files to %s" % (len(files), upload_name)
+for file in files:
+ file_data = [('file', (file, open(file, 'rb')))]
+ r = aptly_session().post(upload_url, files=file_data)
+ if r.status_code == requests.codes.ok:
+ print "Upload successful: %s" % (file)
+ else:
+ raise StandardError, "Upload failed for %s - http status: %s - message:\n%s" % (upload_name, r.status_code, r.text[:30])
+
+file_data = [('file', ('upload.json', json.dumps(upload_meta)))]
+r = aptly_session().post(upload_url, files=file_data)
+if r.status_code == requests.codes.ok:
+ print 'Metadata upload successful.'
+else:
+ raise StandardError, "Upload metadata failed for %s - http status: %s - message:\n%s" % (upload_name, r.status_code, r.text[:30])
--
GitLab