From 629edd034e444fab8aa1204985f85b24fa300ffa Mon Sep 17 00:00:00 2001 From: Ali <> Date: Mon, 8 Aug 2022 19:35:47 +0400 Subject: [PATCH] Modern build script --- buildbox/build.py | 358 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 358 insertions(+) create mode 100644 buildbox/build.py diff --git a/buildbox/build.py b/buildbox/build.py new file mode 100644 index 0000000000..3f66960d07 --- /dev/null +++ b/buildbox/build.py @@ -0,0 +1,358 @@ +#!/usr/bin/python3 + +import argparse +import json +import os +import sys +import shlex +import shutil +import subprocess +import time + +from darwin_containers import DarwinContainers + +def get_clean_env(): + clean_env = os.environ.copy() + return clean_env + +def resolve_executable(program): + def is_executable(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + for path in get_clean_env()["PATH"].split(os.pathsep): + executable_file = os.path.join(path, program) + if is_executable(executable_file): + return executable_file + return None + + +def run_executable_with_output(path, arguments): + executable_path = resolve_executable(path) + if executable_path is None: + raise Exception('Could not resolve {} to a valid executable file'.format(path)) + + process = subprocess.Popen( + [executable_path] + arguments, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=get_clean_env() + ) + output_data, _ = process.communicate() + output_string = output_data.decode('utf-8') + return output_string + +def session_scp_upload(session, source_path, destination_path): + scp_command = 'scp -i {privateKeyPath} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr {source_path} containerhost@"{ipAddress}":{destination_path}'.format( + privateKeyPath=session.privateKeyPath, + ipAddress=session.ipAddress, + source_path=shlex.quote(source_path), + destination_path=shlex.quote(destination_path) + ) + if os.system(scp_command) != 0: + print('Command {} finished with a non-zero status'.format(scp_command)) + +def session_ssh(session, command): + ssh_command = 'ssh -i {privateKeyPath} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null containerhost@"{ipAddress}" -o ServerAliveInterval=60 -t "{command}"'.format( + privateKeyPath=session.privateKeyPath, + ipAddress=session.ipAddress, + command=command + ) + return os.system(ssh_command) + +def remote_build(darwin_containers_host, configuration): + base_dir = os.getcwd() + + configuration_path = 'versions.json' + xcode_version = '' + with open(configuration_path) as file: + configuration_dict = json.load(file) + if configuration_dict['xcode'] is None: + raise Exception('Missing xcode version in {}'.format(configuration_path)) + xcode_version = configuration_dict['xcode'] + + print('Xcode version: {}'.format(xcode_version)) + + commit_count = run_executable_with_output('git', [ + 'rev-list', + '--count', + 'HEAD' + ]) + + build_number_offset = 0 + with open('build_number_offset') as file: + build_number_offset = int(file.read()) + + build_number = build_number_offset + int(commit_count) + print('Build number: {}'.format(build_number)) + + macos_version = '12.5' + image_name = 'macos-{macos_version}-xcode-{xcode_version}'.format(macos_version=macos_version, xcode_version=xcode_version) + + print('Image name: {}'.format(image_name)) + + buildbox_dir = 'buildbox' + os.makedirs('{buildbox_dir}/transient-data'.format(buildbox_dir=buildbox_dir), exist_ok=True) + + codesigning_subpath = '' + remote_configuration = '' + if configuration == 'appcenter': + remote_configuration = 'hockeyapp' + elif configuration == 'appstore': + remote_configuration = 'appstore' + elif configuration == 'reproducible': + codesigning_subpath = 'build-system/fake-codesigning' + remote_configuration = 'verify' + + destination_codesigning_path = '{buildbox_dir}/transient-data/telegram-codesigning'.format(buildbox_dir=buildbox_dir) + destination_build_configuration_path = '{buildbox_dir}/transient-data/build-configuration'.format(buildbox_dir=buildbox_dir) + + if os.path.exists(destination_codesigning_path): + shutil.rmtree(destination_codesigning_path) + if os.path.exists(destination_build_configuration_path): + shutil.rmtree(destination_build_configuration_path) + + shutil.copytree('build-system/fake-codesigning', '{buildbox_dir}/transient-data/telegram-codesigning'.format(buildbox_dir=buildbox_dir)) + shutil.copytree('build-system/example-configuration', '{buildbox_dir}/transient-data/build-configuration'.format(buildbox_dir=buildbox_dir)) + else: + print('Unknown configuration {}'.format(configuration)) + sys.exit(1) + + source_dir = os.path.basename(base_dir) + source_archive_path = '{buildbox_dir}/transient-data/source.tar'.format(buildbox_dir=buildbox_dir) + + if os.path.exists(source_archive_path): + os.remove(source_archive_path) + + print('Compressing source code...') + os.system('find . -type f -a -not -regex "\\." -a -not -regex ".*\\./git" -a -not -regex ".*\\./git/.*" -a -not -regex "\\./bazel-bin" -a -not -regex "\\./bazel-bin/.*" -a -not -regex "\\./bazel-out" -a -not -regex "\\./bazel-out/.*" -a -not -regex "\\./bazel-testlogs" -a -not -regex "\\./bazel-testlogs/.*" -a -not -regex "\\./bazel-telegram-ios" -a -not -regex "\\./bazel-telegram-ios/.*" -a -not -regex "\\./buildbox" -a -not -regex "\\./buildbox/.*" -a -not -regex "\\./buck-out" -a -not -regex "\\./buck-out/.*" -a -not -regex "\\./\\.buckd" -a -not -regex "\\./\\.buckd/.*" -a -not -regex "\\./build" -a -not -regex "\\./build/.*" -print0 | tar cf "{buildbox_dir}/transient-data/source.tar" --null -T -'.format(buildbox_dir=buildbox_dir)) + + darwinContainers = DarwinContainers(serverAddress=darwin_containers_host, verbose=False) + + print('Opening container session...') + with darwinContainers.workingImageSession(name=image_name) as session: + print('Uploading data to container...') + session_scp_upload(session=session, source_path=codesigning_subpath, destination_path='codesigning_data') + session_scp_upload(session=session, source_path='{base_dir}/{buildbox_dir}/transient-data/build-configuration'.format(base_dir=base_dir, buildbox_dir=buildbox_dir), destination_path='telegram-configuration') + session_scp_upload(session=session, source_path='{base_dir}/{buildbox_dir}/guest-build-telegram.sh'.format(base_dir=base_dir, buildbox_dir=buildbox_dir), destination_path='') + session_scp_upload(session=session, source_path='{base_dir}/{buildbox_dir}/transient-data/source.tar'.format(base_dir=base_dir, buildbox_dir=buildbox_dir), destination_path='') + + print('Executing remote build...') + + bazel_cache_host='' + session_ssh(session=session, command='BUILD_NUMBER="{build_number}" BAZEL_HTTP_CACHE_URL="{bazel_cache_host}" bash -l guest-build-telegram.sh {remote_configuration}'.format( + build_number=build_number, + bazel_cache_host=bazel_cache_host, + remote_configuration=remote_configuration + )) + + print('Retrieving build artifacts...') + + artifacts_path='{base_dir}/build/artifacts'.format(base_dir=base_dir) + if os.path.exists(artifacts_path): + shutil.rmtree(artifacts_path) + os.makedirs(artifacts_path, exist_ok=True) + + session_scp_download(session=session, source_path='telegram-ios/build/artifacts/*', destination_path='{artifacts_path}/'.format(artifacts_path=artifacts_path)) + print('Artifacts have been stored at {}'.format(artifacts_path)) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(prog='build') + + parser.add_argument( + '--verbose', + action='store_true', + default=False, + help='Print debug info' + ) + + subparsers = parser.add_subparsers(dest='commandName', help='Commands') + + remote_build_parser = subparsers.add_parser('remote-build', help='Build the app using a remote environment.') + remote_build_parser.add_argument( + '--darwinContainersHost', + required=True, + type=str, + help='DarwinContainers host address.' + ) + remote_build_parser.add_argument( + '--configuration', + choices=[ + 'appcenter', + 'appstore', + 'reproducible' + ], + required=True, + help='Build configuration' + ) + + if len(sys.argv) < 2: + parser.print_help() + sys.exit(1) + + args = parser.parse_args() + + if args.commandName is None: + exit(0) + + if args.commandName == 'remote-build': + remote_build(darwin_containers_host=args.darwinContainersHost, configuration=args.configuration) + + +'''set -e + +rm -f "tools/bazel" +cp "$BAZEL" "tools/bazel" + +BUILD_CONFIGURATION="$1" + +if [ "$BUILD_CONFIGURATION" == "hockeyapp" ] || [ "$BUILD_CONFIGURATION" == "appcenter-experimental" ] || [ "$BUILD_CONFIGURATION" == "appcenter-experimental-2" ]; then + CODESIGNING_SUBPATH="$BUILDBOX_DIR/transient-data/telegram-codesigning/codesigning" +elif [ "$BUILD_CONFIGURATION" == "appstore" ] || [ "$BUILD_CONFIGURATION" == "appstore-development" ]; then + CODESIGNING_SUBPATH="$BUILDBOX_DIR/transient-data/telegram-codesigning/codesigning" +elif [ "$BUILD_CONFIGURATION" == "verify" ]; then + CODESIGNING_SUBPATH="build-system/fake-codesigning" +else + echo "Unknown configuration $1" + exit 1 +fi + +COMMIT_COMMENT="$(git log -1 --pretty=%B)" +case "$COMMIT_COMMENT" in + *"[nocache]"*) + export BAZEL_HTTP_CACHE_URL="" + ;; +esac + +COMMIT_ID="$(git rev-parse HEAD)" +COMMIT_AUTHOR=$(git log -1 --pretty=format:'%an') +if [ -z "$2" ]; then + COMMIT_COUNT=$(git rev-list --count HEAD) + BUILD_NUMBER_OFFSET="$(cat build_number_offset)" + COMMIT_COUNT="$(($COMMIT_COUNT+$BUILD_NUMBER_OFFSET))" + BUILD_NUMBER="$COMMIT_COUNT" +else + BUILD_NUMBER="$2" +fi + +BASE_DIR=$(pwd) + +if [ "$BUILD_CONFIGURATION" == "hockeyapp" ] || [ "$BUILD_CONFIGURATION" == "appcenter-experimental" ] || [ "$BUILD_CONFIGURATION" == "appcenter-experimental-2" ] || [ "$BUILD_CONFIGURATION" == "appstore" ] || [ "$BUILD_CONFIGURATION" == "appstore-development" ]; then + if [ ! `which generate-configuration.sh` ]; then + echo "generate-configuration.sh not found in PATH $PATH" + exit 1 + fi + + mkdir -p "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning" + mkdir -p "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration" + + case "$BUILD_CONFIGURATION" in + "hockeyapp"|"appcenter-experimental"|"appcenter-experimental-2") + generate-configuration.sh internal release "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning" "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration" + ;; + + "appstore") + generate-configuration.sh appstore release "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning" "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration" + ;; + + "appstore-development") + generate-configuration.sh appstore development "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning" "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration" + ;; + + *) + echo "Unknown build configuration $BUILD_CONFIGURATION" + exit 1 + ;; + esac +elif [ "$BUILD_CONFIGURATION" == "verify" ]; then + mkdir -p "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning" + mkdir -p "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration" + + cp -R build-system/fake-codesigning/* "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning/" + cp -R build-system/example-configuration/* "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration/" +fi + +if [ ! -d "$CODESIGNING_SUBPATH" ]; then + echo "$CODESIGNING_SUBPATH does not exist" + exit 1 +fi + +SOURCE_DIR=$(basename "$BASE_DIR") +rm -f "$BUILDBOX_DIR/transient-data/source.tar" +set -x +find . -type f -a -not -regex "\\." -a -not -regex ".*\\./git" -a -not -regex ".*\\./git/.*" -a -not -regex "\\./bazel-bin" -a -not -regex "\\./bazel-bin/.*" -a -not -regex "\\./bazel-out" -a -not -regex "\\./bazel-out/.*" -a -not -regex "\\./bazel-testlogs" -a -not -regex "\\./bazel-testlogs/.*" -a -not -regex "\\./bazel-telegram-ios" -a -not -regex "\\./bazel-telegram-ios/.*" -a -not -regex "\\./buildbox" -a -not -regex "\\./buildbox/.*" -a -not -regex "\\./buck-out" -a -not -regex "\\./buck-out/.*" -a -not -regex "\\./\\.buckd" -a -not -regex "\\./\\.buckd/.*" -a -not -regex "\\./build" -a -not -regex "\\./build/.*" -print0 | tar cf "$BUILDBOX_DIR/transient-data/source.tar" --null -T - + +PROCESS_ID="$$" + +if [ -z "$RUNNING_VM" ]; then + VM_NAME="$VM_BASE_NAME-$(openssl rand -hex 10)-build-telegram-$PROCESS_ID" +else + VM_NAME="$RUNNING_VM" +fi + +if [ "$BUILD_MACHINE" == "linux" ]; then + virt-clone --original "$VM_BASE_NAME" --name "$VM_NAME" --auto-clone + virsh start "$VM_NAME" + + echo "Getting VM IP" + + while [ 1 ]; do + TEST_IP=$(virsh domifaddr "$VM_NAME" 2>/dev/null | egrep -o 'ipv4.*' | sed -e 's/ipv4\s*//g' | sed -e 's|/.*||g') + if [ ! -z "$TEST_IP" ]; then + RESPONSE=$(ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null telegram@"$TEST_IP" -o ServerAliveInterval=60 -t "echo -n 1") + if [ "$RESPONSE" == "1" ]; then + VM_IP="$TEST_IP" + break + fi + fi + sleep 1 + done +elif [ "$BUILD_MACHINE" == "macOS" ]; then + if [ -z "$RUNNING_VM" ]; then + prlctl clone "$VM_BASE_NAME" --linked --name "$VM_NAME" + prlctl start "$VM_NAME" + + echo "Getting VM IP" + + while [ 1 ]; do + TEST_IP=$(prlctl exec "$VM_NAME" "ifconfig | grep inet | grep broadcast | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' | head -1 | tr '\n' '\0'" 2>/dev/null || echo "") + if [ ! -z "$TEST_IP" ]; then + RESPONSE=$(ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null telegram@"$TEST_IP" -o ServerAliveInterval=60 -t "echo -n 1") + if [ "$RESPONSE" == "1" ]; then + VM_IP="$TEST_IP" + break + fi + fi + sleep 1 + done + fi + echo "VM_IP=$VM_IP" +fi + +scp -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr "$CODESIGNING_SUBPATH" telegram@"$VM_IP":codesigning_data +scp -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration" telegram@"$VM_IP":telegram-configuration + +scp -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr "$BUILDBOX_DIR/guest-build-telegram.sh" "$BUILDBOX_DIR/transient-data/source.tar" telegram@"$VM_IP": + +ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null telegram@"$VM_IP" -o ServerAliveInterval=60 -t "export BUILD_NUMBER=\"$BUILD_NUMBER\"; export BAZEL_HTTP_CACHE_URL=\"$BAZEL_HTTP_CACHE_URL\"; $GUEST_SHELL -l guest-build-telegram.sh $BUILD_CONFIGURATION" || true + +OUTPUT_PATH="build/artifacts" +rm -rf "$OUTPUT_PATH" +mkdir -p "$OUTPUT_PATH" + +scp -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr telegram@"$VM_IP":"telegram-ios/build/artifacts/*" "$OUTPUT_PATH/" + +if [ -z "$RUNNING_VM" ]; then + if [ "$BUILD_MACHINE" == "linux" ]; then + virsh destroy "$VM_NAME" + virsh undefine "$VM_NAME" --remove-all-storage --nvram + elif [ "$BUILD_MACHINE" == "macOS" ]; then + echo "Deleting VM..." + #prlctl stop "$VM_NAME" --kill + #prlctl delete "$VM_NAME" + fi +fi + +if [ ! -f "$OUTPUT_PATH/Telegram.ipa" ]; then + exit 1 +fi +''' \ No newline at end of file