Update build scripts

build-system/Make/GenerateProfiles.py

@@ -0,0 +1,44 @@
+import json
+import os
+import sys
+import shutil
+import tempfile
+import plistlib
+import argparse
+
+from BuildEnvironment import run_executable_with_output, check_run_system
+
+
+def get_certificate_base64():
+    certificate_data = run_executable_with_output('security', arguments=['find-certificate', '-c', 'Apple Distribution: Telegram FZ-LLC (C67CF9S4VU)', '-p'])
+    certificate_data = certificate_data.replace('-----BEGIN CERTIFICATE-----', '')
+    certificate_data = certificate_data.replace('-----END CERTIFICATE-----', '')
+    certificate_data = certificate_data.replace('\n', '')
+    return certificate_data
+
+
+def process_provisioning_profile(source, destination, certificate_data):
+    parsed_plist = run_executable_with_output('security', arguments=['cms', '-D', '-i', source], check_result=True)
+    parsed_plist_file = tempfile.mktemp()
+    with open(parsed_plist_file, 'w+') as file:
+        file.write(parsed_plist)
+
+    run_executable_with_output('plutil', arguments=['-remove', 'DeveloperCertificates.0', parsed_plist_file])
+    run_executable_with_output('plutil', arguments=['-insert', 'DeveloperCertificates.0', '-data', certificate_data, parsed_plist_file])
+    run_executable_with_output('plutil', arguments=['-remove', 'DER-Encoded-Profile', parsed_plist_file])
+
+    run_executable_with_output('security', arguments=['cms', '-S', '-N', 'Apple Distribution: Telegram FZ-LLC (C67CF9S4VU)', '-i', parsed_plist_file, '-o', destination])    
+
+    os.unlink(parsed_plist_file)
+
+
+def generate_provisioning_profiles(source_path, destination_path):
+    certificate_data = get_certificate_base64()
+
+    if not os.path.exists(destination_path):
+        print('{} does not exits'.format(destination_path))
+        sys.exit(1)
+
+    for file_name in os.listdir(source_path):
+        if file_name.endswith('.mobileprovision'):
+            process_provisioning_profile(source=source_path + '/' + file_name, destination=destination_path + '/' + file_name, certificate_data=certificate_data)

build-system/Make/Make.py

@@ -14,6 +14,7 @@ from ProjectGeneration import generate
 from BazelLocation import locate_bazel
 from BuildConfiguration import CodesigningSource, GitCodesigningSource, DirectoryCodesigningSource, BuildConfiguration, build_configuration_from_json
 import RemoteBuild
+import GenerateProfiles
 
 
 class ResolvedCodesigningData:
@@ -920,6 +921,12 @@ if __name__ == '__main__':
         type=str,
         help='DarwinContainers host address.'
     )
+    remote_build_parser.add_argument(
+        '--darwinContainers',
+        required=True,
+        type=str,
+        help='DarwinContainers script path.'
+    )
     remote_build_parser.add_argument(
         '--configuration',
         choices=[
@@ -940,6 +947,15 @@ if __name__ == '__main__':
         help='Bazel remote cache host address.'
     )
 
+    generate_profiles_build_parser = subparsers.add_parser('generate-verification-profiles', help='Generate provisioning profiles that can be used to build a veritication IPA.')
+    add_codesigning_common_arguments(generate_profiles_build_parser)
+    generate_profiles_build_parser.add_argument(
+        '--destination',
+        required=True,
+        type=str,
+        help='Path to the destination directory.'
+    )
+
     remote_upload_testflight_parser = subparsers.add_parser('remote-deploy-testflight', help='Build the app using a remote environment.')
     remote_upload_testflight_parser.add_argument(
         '--darwinContainersHost',
@@ -947,6 +963,12 @@ if __name__ == '__main__':
         type=str,
         help='DarwinContainers host address.'
     )
+    remote_upload_testflight_parser.add_argument(
+        '--darwinContainers',
+        required=True,
+        type=str,
+        help='DarwinContainers script path.'
+    )
     remote_upload_testflight_parser.add_argument(
         '--ipa',
         required=True,
@@ -967,6 +989,12 @@ if __name__ == '__main__':
         type=str,
         help='DarwinContainers host address.'
     )
+    remote_ipadiff_parser.add_argument(
+        '--darwinContainers',
+        required=True,
+        type=str,
+        help='DarwinContainers script path.'
+    )
     remote_ipadiff_parser.add_argument(
         '--ipa1',
         required=True,
@@ -1024,11 +1052,33 @@ if __name__ == '__main__':
             shutil.copyfile(args.configurationPath, remote_input_path + '/configuration.json')
 
             RemoteBuild.remote_build(
+                darwin_containers_path=args.darwinContainers,
                 darwin_containers_host=args.darwinContainersHost,
                 bazel_cache_host=args.cacheHost,
                 configuration=args.configuration,
                 build_input_data_path=remote_input_path
             )
+        elif args.commandName == 'generate-verification-profiles':
+            base_path = os.getcwd()
+            remote_input_path = '{}/build-input/remote-input'.format(base_path)
+            if os.path.exists(remote_input_path):
+                shutil.rmtree(remote_input_path)
+            os.makedirs(remote_input_path)
+            os.makedirs(remote_input_path + '/certs')
+            os.makedirs(remote_input_path + '/profiles')
+
+            if os.path.exists(args.destination):
+                shutil.rmtree(args.destination)
+            os.makedirs(args.destination)
+
+            resolve_configuration(
+                base_path=os.getcwd(),
+                bazel_command_line=None,
+                arguments=args,
+                additional_codesigning_output_path=remote_input_path
+            )
+
+            GenerateProfiles.generate_provisioning_profiles(source_path=remote_input_path + '/profiles', destination_path=args.destination)
         elif args.commandName == 'remote-deploy-testflight':
             env = os.environ
             if 'APPSTORE_CONNECT_USERNAME' not in env:
@@ -1039,6 +1089,7 @@ if __name__ == '__main__':
                 sys.exit(1)
 
             RemoteBuild.remote_deploy_testflight(
+                darwin_containers_path=args.darwinContainers,
                 darwin_containers_host=args.darwinContainersHost,
                 ipa_path=args.ipa,
                 dsyms_path=args.dsyms,
@@ -1047,6 +1098,7 @@ if __name__ == '__main__':
             )
         elif args.commandName == 'remote-ipa-diff':
             RemoteBuild.remote_ipa_diff(
+                darwin_containers_path=args.darwinContainers,
                 darwin_containers_host=args.darwinContainersHost,
                 ipa1_path=args.ipa1,
                 ipa2_path=args.ipa2

build-system/Make/RemoteBuild.py

@@ -4,41 +4,61 @@ import json
 import shutil
 import shlex
 import tempfile
+import importlib.util
+from importlib.machinery import SourceFileLoader
 
 from BuildEnvironment import run_executable_with_output
 
+
+def import_module_from_file(module_name, file_path):
+    if not os.path.exists(file_path):
+        print('{} does not exist'.format(file_path))
+        sys.exit(1)
+
+    loader = SourceFileLoader(module_name, file_path)
+    spec = importlib.util.spec_from_file_location(module_name, loader=loader)
+    module = importlib.util.module_from_spec(spec)
+
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
 def session_scp_upload(session, source_path, destination_path):
     scp_command = 'scp -i {privateKeyPath} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr {source_path} containerhost@"{ipAddress}":{destination_path}'.format(
-        privateKeyPath=session.privateKeyPath,
-        ipAddress=session.ipAddress,
+        privateKeyPath=session.private_key_path,
+        ipAddress=session.ip_address,
         source_path=shlex.quote(source_path),
         destination_path=shlex.quote(destination_path)
     )
     if os.system(scp_command) != 0:
         print('Command {} finished with a non-zero status'.format(scp_command))
 
+
 def session_scp_download(session, source_path, destination_path):
     scp_command = 'scp -i {privateKeyPath} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr containerhost@"{ipAddress}":{source_path} {destination_path}'.format(
-        privateKeyPath=session.privateKeyPath,
-        ipAddress=session.ipAddress,
+        privateKeyPath=session.private_key_path,
+        ipAddress=session.ip_address,
         source_path=shlex.quote(source_path),
         destination_path=shlex.quote(destination_path)
     )
     if os.system(scp_command) != 0:
         print('Command {} finished with a non-zero status'.format(scp_command))
 
+
 def session_ssh(session, command):
     ssh_command = 'ssh -i {privateKeyPath} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null containerhost@"{ipAddress}" -o ServerAliveInterval=60 -t "{command}"'.format(
-        privateKeyPath=session.privateKeyPath,
-        ipAddress=session.ipAddress,
+        privateKeyPath=session.private_key_path,
+        ipAddress=session.ip_address,
         command=command
     )
     return os.system(ssh_command)
 
-def remote_build(darwin_containers_host, bazel_cache_host, configuration, build_input_data_path):
-    macos_version = '12.5'
 
-    from darwin_containers import DarwinContainers
+def remote_build(darwin_containers_path, darwin_containers_host, bazel_cache_host, configuration, build_input_data_path):
+    macos_version = '13.0'
+
+    DarwinContainers = import_module_from_file('darwin-containers', darwin_containers_path)
 
     base_dir = os.getcwd()
 
@@ -83,67 +103,88 @@ def remote_build(darwin_containers_host, bazel_cache_host, configuration, build_
     print('Compressing source code...')
     os.system('find . -type f -a -not -regex "\\." -a -not -regex ".*\\./git" -a -not -regex ".*\\./git/.*" -a -not -regex "\\./bazel-bin" -a -not -regex "\\./bazel-bin/.*" -a -not -regex "\\./bazel-out" -a -not -regex "\\./bazel-out/.*" -a -not -regex "\\./bazel-testlogs" -a -not -regex "\\./bazel-testlogs/.*" -a -not -regex "\\./bazel-telegram-ios" -a -not -regex "\\./bazel-telegram-ios/.*" -a -not -regex "\\./buildbox" -a -not -regex "\\./buildbox/.*" -a -not -regex "\\./buck-out" -a -not -regex "\\./buck-out/.*" -a -not -regex "\\./\\.buckd" -a -not -regex "\\./\\.buckd/.*" -a -not -regex "\\./build" -a -not -regex "\\./build/.*" -print0 | tar cf "{buildbox_dir}/transient-data/source.tar" --null -T -'.format(buildbox_dir=buildbox_dir))
 
-    darwinContainers = DarwinContainers(serverAddress=darwin_containers_host, verbose=False)
-
     print('Opening container session...')
-    with darwinContainers.workingImageSession(name=image_name) as session:
-        print('Uploading data to container...')
-        session_scp_upload(session=session, source_path=build_input_data_path, destination_path='telegram-build-input')
-        session_scp_upload(session=session, source_path='{base_dir}/{buildbox_dir}/transient-data/source.tar'.format(base_dir=base_dir, buildbox_dir=buildbox_dir), destination_path='')
 
-        guest_build_sh = '''
-            set -x
-            set -e
+    def handle_ssh_credentials(credentials):
+        with DarwinContainers.ContainerSession(credentials=credentials) as session:
+            print('Uploading data to container...')
 
-            mkdir /Users/Shared/telegram-ios
-            cd /Users/Shared/telegram-ios
+            session_scp_upload(session=session, source_path=build_input_data_path, destination_path='telegram-build-input')
+            session_scp_upload(session=session, source_path='{base_dir}/{buildbox_dir}/transient-data/source.tar'.format(base_dir=base_dir, buildbox_dir=buildbox_dir), destination_path='')
 
-            tar -xf $HOME/source.tar
+            guest_build_sh = '''
+                set -x
+                set -e
 
-            python3 build-system/Make/ImportCertificates.py --path $HOME/telegram-build-input/certs
+                mkdir /Users/Shared/telegram-ios
+                cd /Users/Shared/telegram-ios
 
-        '''
+                tar -xf $HOME/source.tar
 
-        guest_build_sh += 'python3 build-system/Make/Make.py \\'
-        if bazel_cache_host is not None:
-            guest_build_sh += '--cacheHost="{}" \\'.format(bazel_cache_host)
-        guest_build_sh += 'build \\'
-        guest_build_sh += ''
-        guest_build_sh += '--buildNumber={} \\'.format(build_number)
-        guest_build_sh += '--configuration={} \\'.format(configuration)
-        guest_build_sh += '--configurationPath=$HOME/telegram-build-input/configuration.json \\'
-        guest_build_sh += '--codesigningInformationPath=$HOME/telegram-build-input \\'
-        guest_build_sh += '--outputBuildArtifactsPath=/Users/Shared/telegram-ios/build/artifacts \\'
+                python3 build-system/Make/ImportCertificates.py --path $HOME/telegram-build-input/certs
 
-        guest_build_file_path = tempfile.mktemp()
-        with open(guest_build_file_path, 'w+') as file:
-            file.write(guest_build_sh)
-        session_scp_upload(session=session, source_path=guest_build_file_path, destination_path='guest-build-telegram.sh')
-        os.unlink(guest_build_file_path)
+            '''
 
-        print('Executing remote build...')
+            guest_build_sh += 'python3 build-system/Make/Make.py \\'
+            if bazel_cache_host is not None:
+                guest_build_sh += '--cacheHost="{}" \\'.format(bazel_cache_host)
+            guest_build_sh += 'build \\'
+            guest_build_sh += ''
+            guest_build_sh += '--buildNumber={} \\'.format(build_number)
+            guest_build_sh += '--configuration={} \\'.format(configuration)
+            guest_build_sh += '--configurationPath=$HOME/telegram-build-input/configuration.json \\'
+            guest_build_sh += '--codesigningInformationPath=$HOME/telegram-build-input \\'
+            guest_build_sh += '--outputBuildArtifactsPath=/Users/Shared/telegram-ios/build/artifacts \\'
 
-        session_ssh(session=session, command='bash -l guest-build-telegram.sh')
+            guest_build_file_path = tempfile.mktemp()
+            with open(guest_build_file_path, 'w+') as file:
+                file.write(guest_build_sh)
+            session_scp_upload(session=session, source_path=guest_build_file_path, destination_path='guest-build-telegram.sh')
+            os.unlink(guest_build_file_path)
 
-        print('Retrieving build artifacts...')
+            print('Executing remote build...')
 
-        artifacts_path='{base_dir}/build/artifacts'.format(base_dir=base_dir)
-        if os.path.exists(artifacts_path):
-            shutil.rmtree(artifacts_path)
-        os.makedirs(artifacts_path, exist_ok=True)
+            session_ssh(session=session, command='bash -l guest-build-telegram.sh')
 
-        session_scp_download(session=session, source_path='/Users/Shared/telegram-ios/build/artifacts/*', destination_path='{artifacts_path}/'.format(artifacts_path=artifacts_path))
+            print('Retrieving build artifacts...')
 
-        if os.path.exists(artifacts_path + '/Telegram.ipa'):
-            print('Artifacts have been stored at {}'.format(artifacts_path))
-        else:
-            print('Telegram.ipa not found')
-            sys.exit(1)
+            artifacts_path='{base_dir}/build/artifacts'.format(base_dir=base_dir)
+            if os.path.exists(artifacts_path):
+                shutil.rmtree(artifacts_path)
+            os.makedirs(artifacts_path, exist_ok=True)
 
-def remote_deploy_testflight(darwin_containers_host, ipa_path, dsyms_path, username, password):
+            session_scp_download(session=session, source_path='/Users/Shared/telegram-ios/build/artifacts/*', destination_path='{artifacts_path}/'.format(artifacts_path=artifacts_path))
+
+            if os.path.exists(artifacts_path + '/Telegram.ipa'):
+                print('Artifacts have been stored at {}'.format(artifacts_path))
+                sys.exit(0)
+            else:
+                print('Telegram.ipa not found')
+                sys.exit(1)
+
+        DarwinContainers.run_remote_ssh(credentials=credentials, command='')
+        sys.exit(0)
+
+    def handle_stopped():
+        pass
+
+    DarwinContainers.DarwinContainers(
+        server_address=darwin_containers_host,
+        verbose=False
+    ).run_image(
+        name=image_name,
+        is_base=False,
+        is_gui=True,
+        is_daemon=False,
+        on_ssh_credentials=handle_ssh_credentials,
+        on_stopped=handle_stopped
+    )
+
+
+def remote_deploy_testflight(darwin_containers_path, darwin_containers_host, ipa_path, dsyms_path, username, password):
     macos_version = '12.5'
 
-    from darwin_containers import DarwinContainers
+    DarwinContainers = import_module_from_file('darwin-containers', darwin_containers_path)
 
     configuration_path = 'versions.json'
     xcode_version = ''
@@ -159,34 +200,50 @@ def remote_deploy_testflight(darwin_containers_host, ipa_path, dsyms_path, usern
 
     print('Image name: {}'.format(image_name))
 
-    darwinContainers = DarwinContainers(serverAddress=darwin_containers_host, verbose=False)
 
-    print('Opening container session...')
-    with darwinContainers.workingImageSession(name=image_name) as session:
-        print('Uploading data to container...')
-        session_scp_upload(session=session, source_path=ipa_path, destination_path='')
-        session_scp_upload(session=session, source_path=dsyms_path, destination_path='')
+    def handle_ssh_credentials(credentials):
+        with DarwinContainers.ContainerSession(credentials=credentials) as session:
+            print('Uploading data to container...')
+            session_scp_upload(session=session, source_path=ipa_path, destination_path='')
+            session_scp_upload(session=session, source_path=dsyms_path, destination_path='')
 
-        guest_upload_sh = '''
-            set -e
+            guest_upload_sh = '''
+                set -e
 
-            export DELIVER_ITMSTRANSPORTER_ADDITIONAL_UPLOAD_PARAMETERS="-t DAV"
-            FASTLANE_PASSWORD="{password}" xcrun altool --upload-app --type ios --file "Telegram.ipa" --username "{username}" --password "@env:FASTLANE_PASSWORD"
-        '''.format(username=username, password=password)
+                export DELIVER_ITMSTRANSPORTER_ADDITIONAL_UPLOAD_PARAMETERS="-t DAV"
+                FASTLANE_PASSWORD="{password}" xcrun altool --upload-app --type ios --file "Telegram.ipa" --username "{username}" --password "@env:FASTLANE_PASSWORD"
+            '''.format(username=username, password=password)
 
-        guest_upload_file_path = tempfile.mktemp()
-        with open(guest_upload_file_path, 'w+') as file:
-            file.write(guest_upload_sh)
-        session_scp_upload(session=session, source_path=guest_upload_file_path, destination_path='guest-upload-telegram.sh')
-        os.unlink(guest_upload_file_path)
+            guest_upload_file_path = tempfile.mktemp()
+            with open(guest_upload_file_path, 'w+') as file:
+                file.write(guest_upload_sh)
+            session_scp_upload(session=session, source_path=guest_upload_file_path, destination_path='guest-upload-telegram.sh')
+            os.unlink(guest_upload_file_path)
 
-        print('Executing remote upload...')
-        session_ssh(session=session, command='bash -l guest-upload-telegram.sh')
+            print('Executing remote upload...')
+            session_ssh(session=session, command='bash -l guest-upload-telegram.sh')
+        sys.exit(0)
 
-def remote_ipa_diff(darwin_containers_host, ipa1_path, ipa2_path):
-    macos_version = '12.5'
+    def handle_stopped():
+        pass
 
-    from darwin_containers import DarwinContainers
+    DarwinContainers.DarwinContainers(
+        server_address=darwin_containers_host,
+        verbose=False
+    ).run_image(
+        name=image_name,
+        is_base=False,
+        is_gui=True,
+        is_daemon=False,
+        on_ssh_credentials=handle_ssh_credentials,
+        on_stopped=handle_stopped
+    )
+
+
+def remote_ipa_diff(darwin_containers_path, darwin_containers_host, ipa1_path, ipa2_path):
+    macos_version = '13.0'
+
+    DarwinContainers = import_module_from_file('darwin-containers', darwin_containers_path)
 
     configuration_path = 'versions.json'
     xcode_version = ''
@@ -202,38 +259,53 @@ def remote_ipa_diff(darwin_containers_host, ipa1_path, ipa2_path):
 
     print('Image name: {}'.format(image_name))
 
-    darwinContainers = DarwinContainers(serverAddress=darwin_containers_host, verbose=False)
-
     print('Opening container session...')
-    with darwinContainers.workingImageSession(name=image_name) as session:
-        print('Uploading data to container...')
-        session_scp_upload(session=session, source_path='tools/ipadiff.py', destination_path='ipadiff.py')
-        session_scp_upload(session=session, source_path='tools/main.cpp', destination_path='main.cpp')
-        session_scp_upload(session=session, source_path=ipa1_path, destination_path='ipa1.ipa')
-        session_scp_upload(session=session, source_path=ipa2_path, destination_path='ipa2.ipa')
 
-        guest_upload_sh = '''
-            set -e
+    def handle_ssh_credentials(credentials):
+        with DarwinContainers.ContainerSession(credentials=credentials) as session:
+            print('Uploading data to container...')
+            session_scp_upload(session=session, source_path='tools/ipadiff.py', destination_path='ipadiff.py')
+            session_scp_upload(session=session, source_path='tools/main.cpp', destination_path='main.cpp')
+            session_scp_upload(session=session, source_path=ipa1_path, destination_path='ipa1.ipa')
+            session_scp_upload(session=session, source_path=ipa2_path, destination_path='ipa2.ipa')
 
-            python3 ipadiff.py ipa1.ipa ipa2.ipa
-            echo $? > result.txt
-        '''
+            guest_upload_sh = '''
+                set -e
 
-        guest_upload_file_path = tempfile.mktemp()
-        with open(guest_upload_file_path, 'w+') as file:
-            file.write(guest_upload_sh)
-        session_scp_upload(session=session, source_path=guest_upload_file_path, destination_path='guest-ipa-diff.sh')
-        os.unlink(guest_upload_file_path)
+                python3 ipadiff.py ipa1.ipa ipa2.ipa
+                echo $? > result.txt
+            '''
 
-        print('Executing remote ipa-diff...')
-        session_ssh(session=session, command='bash -l guest-ipa-diff.sh')
-        guest_result_path = tempfile.mktemp()
-        session_scp_download(session=session, source_path='result.txt', destination_path=guest_result_path)
-        guest_result = ''
-        with open(guest_result_path, 'r') as file:
-            guest_result = file.read().rstrip()
-        os.unlink(guest_result_path)
+            guest_upload_file_path = tempfile.mktemp()
+            with open(guest_upload_file_path, 'w+') as file:
+                file.write(guest_upload_sh)
+            session_scp_upload(session=session, source_path=guest_upload_file_path, destination_path='guest-ipa-diff.sh')
+            os.unlink(guest_upload_file_path)
 
-        if guest_result != '0':
-            sys.exit(1)
+            print('Executing remote ipa-diff...')
+            session_ssh(session=session, command='bash -l guest-ipa-diff.sh')
+            guest_result_path = tempfile.mktemp()
+            session_scp_download(session=session, source_path='result.txt', destination_path=guest_result_path)
+            guest_result = ''
+            with open(guest_result_path, 'r') as file:
+                guest_result = file.read().rstrip()
+            os.unlink(guest_result_path)
 
+            if guest_result != '0':
+                sys.exit(1)
+        sys.exit(0)
+
+    def handle_stopped():
+        pass
+
+    DarwinContainers.DarwinContainers(
+        server_address=darwin_containers_host,
+        verbose=False
+    ).run_image(
+        name=image_name,
+        is_base=False,
+        is_gui=True,
+        is_daemon=False,
+        on_ssh_credentials=handle_ssh_credentials,
+        on_stopped=handle_stopped
+    )