#!/usr/bin/env python3 # Copyright (C) 2014 Igalia S.L. # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA from __future__ import print_function from contextlib import closing import argparse import errno import multiprocessing import os import re import shutil import subprocess import tarfile def enum(**enums): return type('Enum', (), enums) class Rule(object): Result = enum(INCLUDE=1, EXCLUDE=2, NO_MATCH=3) def __init__(self, type, pattern): self.type = type self.original_pattern = pattern self.pattern = re.compile(pattern) def test(self, file): if not(self.pattern.search(file)): return Rule.Result.NO_MATCH return self.type class Ruleset(object): _global_rules = None def __init__(self): # By default, accept all files. self.rules = [Rule(Rule.Result.INCLUDE, '.*')] @classmethod def global_rules(cls): if not cls._global_rules: cls._global_rules = Ruleset() return cls._global_rules @classmethod def add_global_rule(cls, rule): cls.global_rules().add_rule(rule) def add_rule(self, rule): self.rules.append(rule) def passes(self, file): allowed = False for rule in self.rules: result = rule.test(file) if result == Rule.Result.NO_MATCH: continue allowed = Rule.Result.INCLUDE == result return allowed class File(object): def __init__(self, source_root, tarball_root): self.source_root = source_root self.tarball_root = tarball_root def should_skip_file(self, path): # Do not skip files explicitly added from the manifest. return False def get_files(self): yield (self.source_root, self.tarball_root) class Directory(object): def __init__(self, source_root, tarball_root): self.source_root = source_root self.tarball_root = tarball_root self.rules = Ruleset() self.files_in_version_control = self.list_files_in_version_control() def add_rule(self, rule): self.rules.add_rule(rule) def get_tarball_path(self, filename): return filename.replace(self.source_root, self.tarball_root, 1) def list_files_in_version_control(self): # FIXME: Only git is supported for now. p = subprocess.Popen(['git', 'ls-tree', '-r', '--name-only', 'HEAD', self.source_root], stdout=subprocess.PIPE, text="ascii") out = p.communicate()[0] if not out: return [] return frozenset(out.splitlines()) def should_skip_file(self, path): return path not in self.files_in_version_control def get_files(self): for root, dirs, files in os.walk(self.source_root): def passes_all_rules(entry): return Ruleset.global_rules().passes(entry) and self.rules.passes(entry) to_keep = list(filter(passes_all_rules, dirs)) del dirs[:] dirs.extend(to_keep) for file in files: file = os.path.join(root, file) if not passes_all_rules(file): continue yield (file, self.get_tarball_path(file)) class Manifest(object): def __init__(self, manifest_filename, source_root, build_root, tarball_root='/'): self.current_directory = None self.directories = [] self.tarball_root = tarball_root self.source_root = source_root self.build_root = build_root # Normalize the tarball root so that it starts and ends with a slash. if not self.tarball_root.endswith('/'): self.tarball_root = self.tarball_root + '/' if not self.tarball_root.startswith('/'): self.tarball_root = '/' + self.tarball_root with open(manifest_filename, 'r') as file: for line in file.readlines(): self.process_line(line) def add_rule(self, rule): if self.current_directory is not None: self.current_directory.add_rule(rule) else: Ruleset.add_global_rule(rule) def add_directory(self, directory): self.current_directory = directory self.directories.append(directory) def get_full_source_path(self, source_path): if not os.path.exists(source_path): source_path = os.path.join(self.source_root, source_path) if not os.path.exists(source_path): raise Exception('Could not find directory %s' % source_path) return source_path def get_full_tarball_path(self, path): return self.tarball_root + path def get_source_and_tarball_paths_from_parts(self, parts): full_source_path = self.get_full_source_path(parts[1]) if len(parts) > 2: full_tarball_path = self.get_full_tarball_path(parts[2]) else: full_tarball_path = self.get_full_tarball_path(parts[1]) return (full_source_path, full_tarball_path) def process_line(self, line): parts = line.split() if not parts: return if parts[0].startswith("#"): return if parts[0] == "directory" and len(parts) > 1: self.add_directory(Directory(*self.get_source_and_tarball_paths_from_parts(parts))) elif parts[0] == "file" and len(parts) > 1: self.add_directory(File(*self.get_source_and_tarball_paths_from_parts(parts))) elif parts[0] == "exclude" and len(parts) > 1: self.add_rule(Rule(Rule.Result.EXCLUDE, parts[1])) elif parts[0] == "include" and len(parts) > 1: self.add_rule(Rule(Rule.Result.INCLUDE, parts[1])) else: raise Exception('Line does not begin with a correct rule:\n\t' + line) def should_skip_file(self, directory, filename): # Only allow files that are not in version control when they are explicitly included in the manifest from the build dir. if filename.startswith(self.build_root): return False return directory.should_skip_file(filename) def get_files(self): for directory in self.directories: for file_tuple in directory.get_files(): if self.should_skip_file(directory, file_tuple[0]): continue yield file_tuple def create_tarfile(self, output): count = 0 for file_tuple in self.get_files(): count = count + 1 with closing(tarfile.open(output, 'w')) as tarball: for i, (file_path, tarball_path) in enumerate(self.get_files(), start=1): print('Tarring file {0} of {1}'.format(i, count).ljust(40), end='\r') tarball.add(file_path, tarball_path) print("Wrote {0}".format(output).ljust(40)) class Distcheck(object): BUILD_DIRECTORY_NAME = "_build" INSTALL_DIRECTORY_NAME = "_install" def __init__(self, source_root, build_root): self.source_root = source_root self.build_root = build_root def extract_tarball(self, tarball_path): with closing(tarfile.open(tarball_path, 'r')) as tarball: tarball.extractall(self.build_root) def configure(self, dist_dir, build_dir, install_dir, port): def create_dir(directory, directory_type): try: os.mkdir(directory) except OSError as e: if e.errno != errno.EEXIST or not os.path.isdir(directory): raise Exception("Could not create %s dir at %s: %s" % (directory_type, directory, str(e))) create_dir(build_dir, "build") create_dir(install_dir, "install") command = ['cmake', '-DPORT=%s' % port, '-DCMAKE_INSTALL_PREFIX=%s' % install_dir, '-DCMAKE_BUILD_TYPE=Release', '-DENABLE_GTKDOC=ON', '-DENABLE_MINIBROWSER=ON', dist_dir] subprocess.check_call(command, cwd=build_dir) def build(self, build_dir): command = ['make'] make_args = os.getenv('MAKE_ARGS') if make_args: command.extend(make_args.split(' ')) else: command.append('-j%d' % multiprocessing.cpu_count()) subprocess.check_call(command, cwd=build_dir) def check_symbols(self, build_dir): check_bss = os.path.join(self.source_root, 'Tools', 'Scripts', 'check-for-global-bss-symbols-in-webkitgtk-libs') libjsc = os.path.join(build_dir, 'lib', 'libjavascriptcoregtk-4.1.so') libwk = os.path.join(build_dir, 'lib', 'libwebkit2gtk-4.1.so') subprocess.check_call([check_bss, libjsc, libwk]) check_version_script = os.path.join(self.source_root, 'Tools', 'Scripts', 'check-for-invalid-symbols-in-version-script') version_script = os.path.join(os.path.dirname(build_dir), 'Source', 'WebKit', 'webkitglib-symbols.map') subprocess.check_call([check_version_script, version_script, libwk]) def install(self, build_dir): subprocess.check_call(['make', 'install'], cwd=build_dir) def clean(self, dist_dir): shutil.rmtree(dist_dir) def check(self, tarball, port): tarball_name, ext = os.path.splitext(os.path.basename(tarball)) dist_dir = os.path.join(self.build_root, tarball_name) build_dir = os.path.join(dist_dir, self.BUILD_DIRECTORY_NAME) install_dir = os.path.join(dist_dir, self.INSTALL_DIRECTORY_NAME) self.extract_tarball(tarball) self.configure(dist_dir, build_dir, install_dir, port) self.build(build_dir) if port == 'GTK': self.check_symbols(build_dir) self.install(build_dir) self.clean(dist_dir) if __name__ == "__main__": class FilePathAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, os.path.abspath(values)) def ensure_version_if_possible(arguments): if arguments.version is not None: return pkgconfig_file = os.path.join(arguments.build_dir, "Source/WebKit/webkit2gtk-4.1.pc") if os.path.isfile(pkgconfig_file): p = subprocess.Popen(['pkg-config', '--modversion', pkgconfig_file], stdout=subprocess.PIPE, text="ascii") version = p.communicate()[0] if version: arguments.version = version.rstrip('\n') def get_tarball_root_and_output_filename_from_arguments(arguments): tarball_root = arguments.tarball_name if arguments.version is not None: tarball_root += '-' + arguments.version output_filename = os.path.join(arguments.build_dir, tarball_root + ".tar") return tarball_root, output_filename parser = argparse.ArgumentParser(description='Build a distribution bundle.') parser.add_argument('-c', '--check', action='store_true', help='Check the tarball') parser.add_argument('-s', '--source-dir', type=str, action=FilePathAction, default=os.getcwd(), help='The top-level directory of the source distribution. ' + \ 'Directory for relative paths. Defaults to current directory.') parser.add_argument('--version', type=str, default=None, help='The version of the tarball to generate') parser.add_argument('-b', '--build-dir', type=str, action=FilePathAction, default=os.getcwd(), help='The top-level path of directory of the build root. ' + \ 'By default is the current directory.') parser.add_argument('-t', '--tarball-name', type=str, default='webkitgtk', help='Base name of tarball. Defaults to "webkitgtk".') parser.add_argument('-p', '--port', type=str, default='GTK', help='Port to be built in tarball check. Defaults to "GTK".') parser.add_argument('manifest_filename', metavar="manifest", type=str, action=FilePathAction, help='The path to the manifest file.') arguments = parser.parse_args() # Paths in the manifest are relative to the source directory, and this script assumes that # current working directory is the source directory, so change the current working directory # to be the source directory. os.chdir(arguments.source_dir) ensure_version_if_possible(arguments) tarball_root, output_filename = get_tarball_root_and_output_filename_from_arguments(arguments) manifest = Manifest(arguments.manifest_filename, arguments.source_dir, arguments.build_dir, tarball_root) manifest.create_tarfile(output_filename) if arguments.check: Distcheck(arguments.source_dir, arguments.build_dir).check(output_filename, arguments.port)