This script creates .files.tar.gz for Arch Linux repositories for pkgtools.
This file will be used by pkgfile of the pkgtools package to provide a list of files which will be installed by the package.
Just change the config variable to your needs and start the scripts with the wanted repository and architecture or all as parameter.
E.g.:
createfiles.py --repo=„i686,x86_64“
or
createfiles.py --all
Dependencies:
Version History:
0.2.1:
0.2:
0.1:
#!/usr/bin/env python ###################################################################### # # Author: Andrwe Lord Weber # E-Mail: lord-weber-andrwe<at>renona-studios<dot>org # Version: 0.2.1 # Description: # This script builds files.tar.gz archives # for repositories of Arch Linux # ###################################################################### import sys, os, getopt, subprocess, atexit, tarfile, re, shutil, subprocess from signal import signal, SIGTERM, SIGINT from random import random # hash array for setting up needed values # path should be without / at the end # structure: # "repository": { # "architecture": "/repository/path", # "architecture2": "/repository/path2" # }, # "repository2": { # } config = { "andrwe": { "i686": "/data/repo/pkgs/i686", "x86_64": "/data/repo/pkgs/x86_64", } } # static variables # extension of database archive of the repositories DBEXT = ".db.tar.gz" # extension of files archive of the repositories FILEXT = ".files.tar.gz" # directory for temporary files TMPDIR = "/tmp/createfiles.py." + str(random()) # binary of xz-utils, if None it is searched in $PATH XZBIN = None ###################################################################### # # After this line no changes are needed # ###################################################################### def usage(): print "" print "" print "This is create_files.py (C) 2010 by Andrwe Lord Weber <lord-weber-andrwe AT renona-studios DOT org>" print "" print "Usage:" print " create_files.py --<repository=\"architecture1,architecture2\"|all>" print "" print "Configured repositories with architectures are:" for repo,archs in config.iteritems(): for arch,path in archs.iteritems(): print " --" + str(repo) + "=" + str(arch) + " with path: " + str(path) sys.exit(0) # check whether program path is correct def _progpath(_program): """Get Full path for given Program""" for _path in os.environ.get('PATH', '').split(':'): if os.path.exists(os.path.join(_path, _program)) and not os.path.isdir(os.path.join(_path, _program)): return os.path.join(_path, _program) return None # return path of program def _get_progbin(_prog, _var): if not _var == None and os.path.exists(_var) and not os.path.isdir(_var): return _var else: return _progpath(_prog) # open tar-file and return tar-object def _get_tarfile_obj(_pkgfile): _filelen = len(_pkgfile) if _pkgfile[_filelen-2:] == "xz": _xzfile = _pkgfile _pkgfile = TMPDIR + "/" + _pkgfile[:_filelen-3].split("/")[-1] if not os.path.exists(_pkgfile): _blaufile = open(_pkgfile, 'w') try: _xzbin = [_get_progbin("xz", XZBIN), "-dc", _xzfile] _suberr = subprocess.Popen(_xzbin, stdout=_blaufile, stderr=subprocess.PIPE).communicate() except: print sys.exc_value sys.exit(1) _blaufile.close() try: tarfile.is_tarfile(_pkgfile) except IOError: print "Error: The file " + _pkgfile + " doesn't exist or isn't readable." sys.exit(1) except: print "Error: An unexpected error occured while running tarfile.is_tarfile in _get_pkginfo() with parameters: " + _pkgfile + " " + _value sys.exit(1) try: _file = tarfile.open(_pkgfile) except CompressionError: print "Error: The compression of " + _pkgfile + " is not supported." sys.exit(1) except: print "Error: An unknown error occured while running tarfile.open in _get_pkginfo() with parameters: " + _pkgfile + " " + _value sys.exit(1) return _file # extract content of .PKGINFO file of tar-object def _get_pkginfo(_file): try: _pkginfo_file = _file.extractfile(".PKGINFO").read() except: print "Error: An unknown error occured while extracting .PKGINFO in _get_pkginfo() with parameters: " + _pkgfile + " " + _value sys.exit(1) return _pkginfo_file # get value of variables in .PKGINFO e.g. pkgname def _get_pkgvalue(pkgfile, value): try: reg = re.compile(value, re.M) match = reg.search(pkgfile) return match.group(1) except: print "Error: An unknown error occured while running re.compile() in _get_pkginfo() with parameters: " + pkgfile + " " + value sys.exit(1) # get filenames of tar-object def _get_pkgfiles(_file): try: _files = _file.getnames() except CompressionError: print "Error: The compression of " + pkgfile + " is not supported." sys.exit(1) except: print "Error: An unknown error occured while running tarfile.open in _get_pkgfiles() with parameters: " + pkgfile sys.exit(1) try: _reg = re.compile("^(\..*)$") _newfiles = [] for _tmpfile in _files: if not _reg.search(_tmpfile): _newfiles.append(_tmpfile + "\n") del _files return _newfiles except: print "Error: An unknown error occured while running re.compile() in _get_pkginfo() with parameters: " + _file sys.exit(1) # builds 'files' file for pkgfile for given repository and architecture def _build_pkgfile(_filepath, repo, arch): _content = ["%FILES%\n"] _tarfile_obj = _get_tarfile_obj(_filepath) _pkgfile = _get_pkginfo(_tarfile_obj) _pkgname = _get_pkgvalue(_pkgfile, "^pkgname = (.*)$") _pkgver = _get_pkgvalue(_pkgfile, "^pkgver = (.*)$") _repodir = repo + "/" + arch _tmppkgdir = TMPDIR + "/" + _repodir + "/" + _pkgname + "-" + _pkgver _pkgfiles = _get_pkgfiles(_tarfile_obj) _content.extend(_pkgfiles) _tarfile_obj.close() try: os.makedirs(_tmppkgdir) try: _pkgfile = open(_tmppkgdir + "/files", 'w') _pkgfile.writelines(_content) _pkgfile.close() except: print sys.exc_info() print "Error: Couldn't create file " + _tmppkgdir + "/files" sys.exit(1) except: print "Error: Couldn't create folder " + _tmppkgdir sys.exit(1) # remove temporary directory after sys.exit() def cleanup(): try: os.rmdir(TMPDIR) finally: sys.exit(0) def main(argv): _repos = ["all"] _all = 0 # compile argument list for repo, archs in config.iteritems(): _repos.append(repo + "=") try: opts, args = getopt.getopt(argv, "", _repos) except getopt.GetoptError, err: print str(err) usage() # check for --all parameter in all given arguments for opt in opts: if opt.count("--all") >= 1: _all = 1 print "Option '--all' found. Package will be build for all repositories." os.mkdir(TMPDIR) if _all == 1: # build files.tar. for each configured repository for repo,archs in config.iteritems(): for arch,path in archs.iteritems(): print "Building " + FILEXT + " for " + repo + "/" + arch try: path = path + "/" pkgfiles = repo + FILEXT repofiles = repo + DBEXT for file in os.listdir(path): try: filepath = path + file if filepath != path + repofiles and filepath != path + repofiles + ".old" and filepath != path + pkgfiles and filepath != path + pkgfiles + ".old" and filepath != path + repo + ".db": _build_pkgfile(filepath, repo[2:], arch) except KeyError: print "Error: The given architecture " + arch + " doesn't exist." sys.exit(1) except: print "Error: An error occured in main->for repo, arg in opts:. Path: " + filepath sys.exit(1) os.chdir(TMPDIR + "/" + repo[2:] + "/" + arch) filesarch = tarfile.open(pkgfiles, 'w:gz') for dir in os.listdir("./"): if not dir == pkgfiles: filesarch.add(dir) filesarch.close() if os.path.isfile(path + pkgfiles): try: if tarfile.is_tarfile(path + pkgfiles): try: shutil.move(path + pkgfiles, path + pkgfiles + ".old") except: print "Error: Couldn't move " + path + pkgfiles + " to " + path + pkgfiles + ".old" sys.exit(1) except: print "Error: Existing file " + path + pkgfiles + " isn't a valid tar archive." sys.exit(1) if os.path.isfile(pkgfiles): try: if tarfile.is_tarfile(pkgfiles): try: shutil.move(pkgfiles, path + pkgfiles) except: print "Error: Couldn't move " + pkgfiles + " which was build by this script to " + path + pkgfiles sys.exit(1) except: print "Error: The archive file " + pkgfiles + " build by this script isn't a valid tar archive." sys.exit(1) except: print "Error: An error occured in main->for repo, arg in opts:" else: # builds files.tar. for each given repository for repo, arg in opts: for arch in arg.split(","): print "Building " + FILEXT + " for " + repo[2:] + "/" + arch path = config[repo[2:]][arch] + "/" pkgfiles = repo[2:] + FILEXT repofiles = repo[2:] + DBEXT for file in os.listdir(path): try: filepath = path + file if filepath != path + repofiles and filepath != path + repofiles + ".old" and filepath != path + pkgfiles and filepath != path + pkgfiles + ".old" and filepath != path + repo[2:] + ".db": _build_pkgfile(filepath, repo[2:], arch) except KeyError: print "Error: The given architecture " + arch + " doesn't exist." sys.exit(1) except: print "Error: An error occured in main->'for repo, arg in opts:' filepath:" + filepath sys.exit(1) os.chdir(TMPDIR + "/" + repo[2:] + "/" + arch) filesarch = tarfile.open(pkgfiles, 'w:gz') for dir in os.listdir("./"): if not dir == pkgfiles: filesarch.add(dir) filesarch.close() if os.path.isfile(path + pkgfiles): try: if tarfile.is_tarfile(path + pkgfiles): try: shutil.move(path + pkgfiles, path + pkgfiles + ".old") except: print "Error: Couldn't move " + path + pkgfiles + " to " + path + pkgfiles + ".old" sys.exit(1) except: print "Error: Existing file " + path + pkgfiles + " isn't a valid tar archive." sys.exit(1) if os.path.isfile(pkgfiles): try: if tarfile.is_tarfile(pkgfiles): try: shutil.move(pkgfiles, path + pkgfiles) except: print "Error: Couldn't move " + pkgfiles + " which was build by this script to " + path + pkgfiles sys.exit(1) except: print "Error: The archive file " + pkgfiles + " build by this script isn't a valid tar archive." sys.exit(1) shutil.rmtree(TMPDIR) # Start main-function if __name__ == "__main__": atexit.register(cleanup) signal(SIGTERM, lambda signum, stack_frame: exit(1)) signal(SIGINT, lambda signum, stack_frame: exit(1)) if len(sys.argv) > 1: main(sys.argv[1:]) else: print "Error: You have to give at least one architecture or '--all' as parameter!!" usage()