openmamba/autodist
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

autodist-git: new tool for syncing rpm repository with new git server at src.openmamba.org

Browse Source
This commit is contained in:
Silvan 2024-01-02 20:18:19 +01:00
parent 5bac010c6d
commit fcdbff1da2
3 changed files with 263 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Makefile
View File

@ -69,16 +69,17 @@ install-dirs:
install-programs: install-programs:
@$(INSTALL_SCRIPT) autodist $(DESTDIR)$(bindir)/autodist @$(INSTALL_SCRIPT) autodist $(DESTDIR)$(bindir)/autodist
@$(INSTALL_SCRIPT) autodist-tool $(DESTDIR)$(bindir)/autodist-tool @$(INSTALL_SCRIPT) autodist-git $(DESTDIR)$(bindir)/autodist-git
@$(INSTALL_SCRIPT) autodist-launcher $(DESTDIR)$(bindir)/autodist-launcher
@$(INSTALL_SCRIPT) autoport $(DESTDIR)$(bindir)/autoport
@$(INSTALL_SCRIPT) autoport-launcher $(DESTDIR)$(bindir)/autoport-launcher
@$(INSTALL_SCRIPT) autoport-chroot $(DESTDIR)$(sbindir)/autoport-chroot
@$(INSTALL_SCRIPT) automaint $(DESTDIR)$(sbindir)/automaint
@$(INSTALL_SCRIPT) autodist-inspect $(DESTDIR)$(bindir)/autodist-inspect @$(INSTALL_SCRIPT) autodist-inspect $(DESTDIR)$(bindir)/autodist-inspect
@$(INSTALL_SCRIPT) autodist-repository $(DESTDIR)$(sbindir)/autodist-repository @$(INSTALL_SCRIPT) autodist-launcher $(DESTDIR)$(bindir)/autodist-launcher
@$(INSTALL_SCRIPT) autodist-upstream-updates $(DESTDIR)$(sbindir)/autodist-upstream-updates @$(INSTALL_SCRIPT) autodist-repository $(DESTDIR)$(bindir)/autodist-repository
@$(INSTALL_SCRIPT) autodist-tool $(DESTDIR)$(bindir)/autodist-tool
@$(INSTALL_SCRIPT) autodist-upstream-updates $(DESTDIR)$(bindir)/autodist-upstream-updates
@$(INSTALL_SCRIPT) autoport $(DESTDIR)$(bindir)/autoport
@$(INSTALL_SCRIPT) autoport-chroot $(DESTDIR)$(bindir)/autoport-chroot
@$(INSTALL_SCRIPT) autoport-fix-environment $(DESTDIR)$(bindir)/autoport-fix-environment @$(INSTALL_SCRIPT) autoport-fix-environment $(DESTDIR)$(bindir)/autoport-fix-environment
@$(INSTALL_SCRIPT) autoport-launcher $(DESTDIR)$(bindir)/autoport-launcher
@$(INSTALL_SCRIPT) automaint $(DESTDIR)$(bindir)/automaint
@$(INSTALL_SCRIPT) etc/autodist/scripts/* $(DESTDIR)$(configdir)/scripts/ @$(INSTALL_SCRIPT) etc/autodist/scripts/* $(DESTDIR)$(configdir)/scripts/
@$(INSTALL_SCRIPT) autodist-cgi $(DESTDIR)$(cgidir)/autodist @$(INSTALL_SCRIPT) autodist-cgi $(DESTDIR)$(cgidir)/autodist
@$(INSTALL_SCRIPT) webbuild/webbuild-cgi $(DESTDIR)$(cgidir)/webbuild @$(INSTALL_SCRIPT) webbuild/webbuild-cgi $(DESTDIR)$(cgidir)/webbuild
@ -93,6 +94,7 @@ install-programs:
install-data: install-data:
@$(INSTALL_DATA) etc/autodist/config $(DESTDIR)$(configdir)/config @$(INSTALL_DATA) etc/autodist/config $(DESTDIR)$(configdir)/config
@$(INSTALL_DATA) etc/autodist/config-git $(DESTDIR)$(configdir)/config-git
@$(INSTALL_DATA) etc/autodist/blacklist $(DESTDIR)$(configdir)/blacklist @$(INSTALL_DATA) etc/autodist/blacklist $(DESTDIR)$(configdir)/blacklist
@$(INSTALL_DATA) etc/autodist/distdb.d/* $(DESTDIR)$(configdir)/distdb.d/ @$(INSTALL_DATA) etc/autodist/distdb.d/* $(DESTDIR)$(configdir)/distdb.d/
@$(INSTALL_DATA) etc/sudoers.d/autoport $(DESTDIR)$(sysconfdir)/sudoers.d/autoport @$(INSTALL_DATA) etc/sudoers.d/autoport $(DESTDIR)$(sysconfdir)/sudoers.d/autoport

241
autodist-git Executable file
View File

@ -0,0 +1,241 @@
#!/usr/bin/python3
import glob
import os
import re
import rpm
import sys
import subprocess
import tempfile
from shutil import copyfile
from pyrpm.spec import Spec, replace_macros
from gitea import *
from git import Repo
from functools import cmp_to_key
from configobj import ConfigObj
cfg = ConfigObj(infile='/etc/autodist/config-git')
gitea = Gitea(cfg["GITEA_URL"], cfg["GITEA_TOKEN"])
org = Organization.request(gitea, cfg["GITEA_ORGANIZATION"])
def commitReleaseFromDir(pkg_info):
src_dir = pkg_info["src"]
if src_dir.endswith(".src.rpm"):
# src is a SRPM, use autospec to extract in a temporary folder
src_temp_dir = tempfile.TemporaryDirectory()
src_dir = src_temp_dir.name
subprocess.run(["autospec", "-x", pkg_info["src"], f'--destdir={src_dir}'],
stdout=subprocess.PIPE)
# Delete binary source archives
for pattern in [
'*.zip','*.tar.bz2','*.tar.xz','*.tar.gz','*.tgz','*.txz','*.iso',
'*.run','*.dll','*.bin','*.jar']:
for filename in glob.glob(f'{src_dir}/{pattern}'):
os.remove(filename)
#print(src_dir)
#for line in sys.stdin:
# line = line.rstrip()
# break
# Parse spec file
spec = Spec.from_file(f'{src_dir}/{pkg_info["name"]}.spec')
#specfiletemp_dir = tempfile.TemporaryDirectory()
#spec = Specfile(f'{src_dir}/{pkg_info["name"]}.spec',
# sourcedir=specfiletemp_dir.name)
# Get used information from specfile
commit_text = ""
header_split = []
for c in spec.changelog.split('\n'):
if c == "":
break
if c[0:2] == "* ":
header_split = c.split(" ")
else:
if commit_text != "":
commit_text += "\n"
if c[0:2] == "- ":
commit_text += c[2:]
else:
commit_text += c
commit_text = f'{commit_text} [release {header_split[-1]};' + \
f'{header_split[1]} {header_split[2]} {header_split[3]} {header_split[4]}]'
spec_description = replace_macros(spec.description, spec)
try:
# Find repository
gitea_repo = org.get_repository(pkg_info['name'])
except NotFoundException:
# Repository does not exist -> create
org.create_repo(
repoName=pkg_info['name'],
description=spec.summary,
private=False,
autoInit=True,
gitignores=None,
# license=spec.license,
# readme=spec.description,
issue_labels=None,
default_branch="main",
)
gitea_repo = org.get_repository(pkg_info['name'])
# Set/update repository description and website url
spec_url = replace_macros(spec.url, spec)
spec_summary = replace_macros(spec.summary, spec)
if gitea_repo.description != spec_summary or gitea_repo.website != spec_url:
gitea_repo.description = spec_summary
gitea_repo.website = spec_url
gitea_repo.commit()
# Clone repo
repo_url = f'{cfg["GITEA_SSH_URL"]}{pkg_info["name"]}.git'
# Clone repository to temporary folder
temp_dir = tempfile.TemporaryDirectory()
repo = Repo.clone_from(repo_url, temp_dir.name)
# Check if tag already exists
new_tag = f'{pkg_info["v"]}-{pkg_info["r"]}'.replace('~','+')
if new_tag in repo.tags:
print(f'Skipping {pkg_info["name"]} release '
f'{pkg_info["e"]}:{pkg_info["v"]}-{pkg_info["r"]}: tag {new_tag} already exists')
return
# Set committer user and email
repo.config_writer().set_value("user", "name", cfg["COMMITTER_USER"]).release()
repo.config_writer().set_value("user", "email", cfg["COMMITTER_EMAIL"]).release()
# Create/update README.md
with open(f'{temp_dir.name}/README.md', "w") as readme_file:
readme_file.write(f"# {pkg_info['name']}\n\n{spec_description}")
repo.index.add(['README.md'])
# Update files
dir_list = os.listdir(src_dir)
for dir_file in dir_list:
copyfile(f'{src_dir}/{dir_file}',
f'{temp_dir.name}/{dir_file}')
# Add/modify files
repo.index.add([dir_file])
# Remove deleted files
temp_dir_list = os.listdir(temp_dir.name)
for temp_dir_file in temp_dir_list:
if temp_dir_file == ".git" or temp_dir_file == "README.md":
continue
if not temp_dir_file in dir_list:
repo.index.remove([temp_dir_file], working_tree = True)
# Commit
print(f'Committing {pkg_info["name"]} release {new_tag}...')
repo.index.commit(commit_text)
origin = repo.remote(name='origin')
origin.push()
# Create and commit tag
repo.create_tag(new_tag)
origin.push(new_tag)
#for line in sys.stdin:
# line = line.rstrip()
# break
temp_dir.cleanup()
def comparePkgInfo(item1, item2):
return rpm.labelCompare(
(str(item1['e']), item1['v'], item1['r']),
(str(item2['e']), item2['v'], item2['r']))
def findAndCommitPackageReleases(pkgname, pkgvr):
pkgs_info = []
# Find from archive dir
dirs = [f for f in os.listdir(f'{cfg["ARCHIVE_DIR"]}/{pkgname[0:1]}') if re.match(f'{pkgname}-[^-]*-[^-]*$', f)]
for dir in dirs:
pkg_dir = f'{cfg["ARCHIVE_DIR"]}/{pkgname[0:1]}/{dir}'
spec = Spec.from_file(f'{pkg_dir}/{pkgname}.spec')
parts = re.split(f'{pkgname}-([^-]*)-([^-]*)$', dir)
epoch = 0 if spec.epoch is None else int(spec.epoch)
version = parts[1]
release = parts[2]
pkgs_info.append(
{'name': pkgname, 'e': epoch, 'v': version, 'r': release, 'src': pkg_dir})
# Find from OLD_DIR
dirs = [f for f in os.listdir(f'{cfg["OLD_DIR"]}') if re.match(f'{pkgname}_[0-9]*.[0-9]*$', f)]
for dir in dirs:
srpms_list = glob.glob(f'{cfg["OLD_DIR"]}/{dir}/{pkgname}-*.src.rpm')
for srpm in srpms_list:
parts = re.split('.*-([^-]*)-([^-]*).src.rpm$', srpm)
result = subprocess.run(['rpm', '-q', '--queryformat=%{epoch}', '-p', srpm],
stdout=subprocess.PIPE).stdout.decode('utf-8')
# result is "(none)" if no Epoch is set
epoch = "0" if len(result) > 2 else result
version = parts[1]
release = parts[2]
pkgs_info.append(
{'name': pkgname, 'e': epoch, 'v': version, 'r': release, 'src': srpm})
# Find from SRPMS_DIR
srpms_list = [f for f in os.listdir(cfg["SRPMS_DIR"]) if re.match(f'{pkgname}-[^-]*-[^-]*$', f)]
for srpm in srpms_list:
src_path = f'{cfg["SRPMS_DIR"]}/{srpm}'
parts = re.split('.*-([^-]*)-([^-]*).src.rpm$', srpm)
result = subprocess.run(['rpm', '-q', '--queryformat=%{epoch}', '-p', src_path],
stdout=subprocess.PIPE).stdout.decode('utf-8')
# result is "(none)" if no Epoch is set
epoch = "0" if len(result) > 2 else result
version = parts[1]
release = parts[2]
pkgs_info.append(
{'name': pkgname, 'e': epoch, 'v': version, 'r': release, 'src': src_path })
# Sort releases
pkgs_info.sort(key=cmp_to_key(comparePkgInfo))
for pkg_info in pkgs_info:
vr = f'{pkg_info["v"]}-{pkg_info["r"]}'
if pkgvr is not None and pkgvr != vr:
continue
#pkg_dir = f'{cfg["ARCHIVE_DIR"]}/{pkgname[0:1]}/{pkgname}-{vr}'
commitReleaseFromDir(pkg_info)
def main():
args = sys.argv[1:]
if len(args) == 0:
from_pkg = "ico"
from_reached = False
dir_list = sorted(filter(os.path.isfile, glob.glob(f'{cfg["SRPMS_DIR"]}/*.src.rpm')))
for dir_file in dir_list:
parts = re.split('.*/([^/]*)-([^-]*)-([^-]*).src.rpm$', dir_file)
pkgname = parts[1]
if not from_reached and pkgname != from_pkg:
continue
from_reached = True
print(f'Processing package {pkgname}...')
findAndCommitPackageReleases(pkgname, None)
else:
if len(args) < 1 or len(args) > 2:
print("Usage: autodist-git [pkgname [version-release]]\n")
exit(1)
pkgname = args[0]
pkgvr = None
if len(args) > 1:
pkgvr = args[1]
findAndCommitPackageReleases(pkgname, pkgvr)
print("All done.\n")
main()

12
etc/autodist/config-git Normal file
View File

@ -0,0 +1,12 @@
REPO_DIR = "/var/ftp/pub/openmamba/base"
ARCHIVE_DIR = "%(REPO_DIR)s/archive"
OLD_DIR = "%(REPO_DIR)s/old"
SRPMS_DIR = "%(REPO_DIR)s/SRPMS.base"
GITEA_URL = "https://src.openmamba.org"
GITEA_SSH_URL = "ssh://gitea@src.openmamba.org:3326/rpms/"
GITEA_TOKEN = "<replace with a valid token>"
GITEA_ORGANIZATION = "rpms"
COMMITTER_USER = "autodist"
COMMITTER_EMAIL = "autodist@openmamba.org"