From 3c844dbeb524e3a4afb7d2ec4ad7229a7b829f8a Mon Sep 17 00:00:00 2001 From: Peter <> Date: Mon, 24 Jun 2019 21:28:09 +0300 Subject: [PATCH] Added scripts for build verification --- buildbox/build-telegram.sh | 11 +- buildbox/guest-build-telegram.sh | 12 +- fastlane/Fastfile | 9 +- tools/ipadiff.py | 316 +++++++++++++++++++++++++++++++ tools/main.cpp | 262 +++++++++++++++++++++++++ 5 files changed, 602 insertions(+), 8 deletions(-) create mode 100644 tools/ipadiff.py create mode 100644 tools/main.cpp diff --git a/buildbox/build-telegram.sh b/buildbox/build-telegram.sh index 9f5fbb86ac..ee74b7a554 100644 --- a/buildbox/build-telegram.sh +++ b/buildbox/build-telegram.sh @@ -25,6 +25,15 @@ else exit 1 fi +COMMIT_ID=$(git rev-parse HEAD) +if [ -z "$2" ]; then + COMMIT_COUNT=$(git rev-list --count HEAD) + COMMIT_COUNT="$(($COMMIT_COUNT+1000))" + BUILD_NUMBER="$COMMIT_COUNT" +else + BUILD_NUMBER="$2" +fi + BASE_DIR=$(pwd) if [ "$BUILD_CONFIGURATION" == "hockeyapp" ] || [ "$BUILD_CONFIGURATION" == "appstore" ]; then @@ -87,7 +96,7 @@ else fi scp -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr "$BUILDBOX_DIR/guest-build-telegram.sh" "$BUILDBOX_DIR/transient-data/source.tar" telegram@"$VM_IP": -ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null telegram@"$VM_IP" -o ServerAliveInterval=60 -t "export TELEGRAM_BUILD_APPSTORE_PASSWORD=\"$TELEGRAM_BUILD_APPSTORE_PASSWORD\"; export TELEGRAM_BUILD_APPSTORE_TEAM_NAME=\"$TELEGRAM_BUILD_APPSTORE_TEAM_NAME\"; bash -l guest-build-telegram.sh $BUILD_CONFIGURATION" || true +ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null telegram@"$VM_IP" -o ServerAliveInterval=60 -t "export TELEGRAM_BUILD_APPSTORE_PASSWORD=\"$TELEGRAM_BUILD_APPSTORE_PASSWORD\"; export TELEGRAM_BUILD_APPSTORE_TEAM_NAME=\"$TELEGRAM_BUILD_APPSTORE_TEAM_NAME\"; export BUILD_NUMBER=\"$BUILD_NUMBER\"; export COMMIT_ID=\"$COMMIT_ID\"; bash -l guest-build-telegram.sh $BUILD_CONFIGURATION" || true if [ "$BUILD_CONFIGURATION" == "appstore" ]; then ARCHIVE_PATH="$HOME/telegram-builds-archive" diff --git a/buildbox/guest-build-telegram.sh b/buildbox/guest-build-telegram.sh index f0241e200b..c08c7da671 100644 --- a/buildbox/guest-build-telegram.sh +++ b/buildbox/guest-build-telegram.sh @@ -1,5 +1,15 @@ #!/bin/sh +if [ -z "BUILD_NUMBER" ]; then + echo "BUILD_NUMBER is not set" + exit 1 +fi + +if [ -z "COMMIT_ID" ]; then + echo "COMMIT_ID is not set" + exit 1 +fi + if [ "$1" == "hockeyapp" ]; then FASTLANE_BUILD_CONFIGURATION="internalhockeyapp" CERTS_PATH="codesigning_data/certs" @@ -68,5 +78,5 @@ else tar -xf "source.tar" cd "$SOURCE_PATH" - FASTLANE_PASSWORD="$FASTLANE_PASSWORD" FASTLANE_ITC_TEAM_NAME="$FASTLANE_ITC_TEAM_NAME" fastlane "$FASTLANE_BUILD_CONFIGURATION" + FASTLANE_PASSWORD="$FASTLANE_PASSWORD" FASTLANE_ITC_TEAM_NAME="$FASTLANE_ITC_TEAM_NAME" fastlane "$FASTLANE_BUILD_CONFIGURATION" build_number:"$BUILD_NUMBER" commit_hash:"$COMMIT_ID" fi diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 5dffc7a870..ed178e3a90 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -16,6 +16,7 @@ app_identifier_llc = [ signing_identity_llc = "iPhone Distribution: Digital Fortress LLC (C67CF9S4VU)" lane :do_build_app do |options| + puts("Building with build number: " + options[:build_number] + ", commit id: " + options[:commit_id]) gym( workspace: "Telegram-iOS.xcworkspace", configuration: options[:configuration], @@ -39,16 +40,12 @@ lane :do_build_app do |options| end lane :build_for_appstore do |options| - commit = last_git_commit - commit_count = sh("git rev-list --count HEAD") - commit_count_int = commit_count.to_i + 1000 - commit_count_string = commit_count_int.to_s do_build_app( configuration: "ReleaseAppStoreLLC", scheme: "Telegram-iOS-AppStoreLLC", export_method: "app-store", - build_number: commit_count_string, - commit_id: commit[:commit_hash], + build_number: options[:build_number], + commit_id: options[:commit_hash], signingCertificate: signing_identity_llc, provisioningProfiles: { base_app_identifier_llc => "match AppStore " + base_app_identifier_llc, diff --git a/tools/ipadiff.py b/tools/ipadiff.py new file mode 100644 index 0000000000..11455dd713 --- /dev/null +++ b/tools/ipadiff.py @@ -0,0 +1,316 @@ +import sys +import os +import glob +import tempfile +import re +import filecmp +import subprocess +from zipfile import ZipFile + + +def get_file_list(dir): + result_files = [] + result_dirs = [] + for root, dirs, files in os.walk(dir, topdown=False): + for name in files: + result_files.append(os.path.relpath(os.path.join(root, name), dir)) + for name in dirs: + result_dirs.append(os.path.relpath(os.path.join(root, name), dir)) + return set(result_dirs), set(result_files) + + +def remove_codesign_dirs(dirs): + result = set() + for dir in dirs: + if dir == 'SC_Info': + continue + if re.match('Watch/.*\\.appex/SC_Info', dir): + continue + if re.match('PlugIns/.*\\.appex/SC_Info', dir): + continue + if re.match('Frameworks/.*\\.framework/SC_Info', dir): + continue + result.add(dir) + return result + + +def remove_codesign_files(files): + result = set() + for f in files: + if f == 'embedded.mobileprovision': + continue + if re.match('.*/.*\\.appex/embedded.mobileprovision', f): + continue + if f == '_CodeSignature/CodeResources': + continue + if f == 'CrackerXI': + continue + if re.match('Watch/.*\\.app/embedded.mobileprovision', f): + continue + if re.match('PlugIns/.*\\.appex/_CodeSignature/CodeResources', f): + continue + if re.match('Frameworks/.*\\.framework/_CodeSignature/CodeResources', f): + continue + if f == 'Frameworks/ModernProto.framework/ModernProto': + continue + result.add(f) + return result + + +def remove_watch_files(files): + result = set() + excluded = set() + for f in files: + if re.match('Watch/.*', f): + excluded.add(f) + else: + result.add(f) + return (result, excluded) + + +def remove_plugin_files(files): + result = set() + excluded = set() + for f in files: + if False and re.match('PlugIns/.*', f): + excluded.add(f) + else: + result.add(f) + return (result, excluded) + + +def remove_asset_files(files): + result = set() + excluded = set() + for f in files: + if re.match('.*\\.car', f): + excluded.add(f) + else: + result.add(f) + return (result, excluded) + + +def remove_nib_files(files): + result = set() + excluded = set() + for f in files: + if re.match('.*\\.nib', f): + excluded.add(f) + else: + result.add(f) + return (result, excluded) + + +def diff_dirs(ipa1, dir1, ipa2, dir2): + only_in_ipa1 = dir1.difference(dir2) + only_in_ipa2 = dir2.difference(dir1) + if len(only_in_ipa1) == 0 and len(only_in_ipa2) == 0: + return + print('Directory structure doesn\'t match in ' + ipa1 + ' and ' + ipa2) + if len(only_in_ipa1) != 0: + print('Directories not present in ' + ipa2) + for dir in only_in_ipa1: + print(' ' + dir) + if len(only_in_ipa2) != 0: + print('Directories not present in ' + ipa1) + for dir in only_in_ipa2: + print(' ' + dir) + + sys.exit(1) + + +def is_binary(file): + out = os.popen('file "' + file + '"').read() + if out.find('Mach-O') == -1: + return False + return True + + +def is_xcconfig(file): + if re.match('.*\\.xcconfig', file): + return True + else: + return False + + +def diff_binaries(tempdir, self_base_path, file1, file2): + diff_app = tempdir + '/main' + if not os.path.isfile(diff_app): + if not os.path.isfile(self_base_path + '/main.cpp'): + print('Could not find ' + self_base_path + '/main.cpp') + sys.exit(1) + subprocess.call(['clang', self_base_path + '/main.cpp', '-lc++', '-o', diff_app]) + if not os.path.isfile(diff_app): + print('Could not compile ' + self_base_path + '/main.cpp') + sys.exit(1) + + result = os.popen(diff_app + ' ' + file1 + ' ' + file2).read().strip() + if result == 'Encrypted': + return 'binary_encrypted' + elif result == 'Equal': + return 'equal' + elif result == 'Not Equal': + return 'not_equal' + else: + print('Unexpected data from binary diff code: ' + result) + sys.exit(1) + + +def is_plist(file1): + if file1.find('.plist') == -1: + return False + return True + + +def diff_plists(file1, file2): + remove_properties = ['UISupportedDevices', 'DTAppStoreToolsBuild', 'MinimumOSVersion', 'BuildMachineOSBuild'] + + clean1_properties = '' + clean2_properties = '' + + with open(os.devnull, 'w') as devnull: + for property in remove_properties: + if not subprocess.call(['plutil', '-extract', property, 'xml1', '-o', '-', file1], stderr=devnull, stdout=devnull): + clean1_properties += ' | plutil -remove ' + property + ' -r -o - -- -' + if not subprocess.call(['plutil', '-extract', property, 'xml1', '-o', '-', file2], stderr=devnull, stdout=devnull): + clean2_properties += ' | plutil -remove ' + property + ' -r -o - -- -' + + data1 = os.popen('plutil -convert xml1 "' + file1 + '" -o -' + clean1_properties).read() + data2 = os.popen('plutil -convert xml1 "' + file2 + '" -o -' + clean2_properties).read() + + if data1 == data2: + return 'equal' + else: + with open('lhs.plist', 'wb') as f: + f.write(str.encode(data1)) + with open('rhs.plist', 'wb') as f: + f.write(str.encode(data2)) + sys.exit(1) + return 'not_equal' + + +def diff_xcconfigs(file1, file2): + with open(file1, 'rb') as f: + data1 = f.read().strip() + with open(file2, 'rb') as f: + data2 = f.read().strip() + if data1 != data2: + return 'not_equal' + return 'equal' + + +def diff_files(ipa1, files1, ipa2, files2): + only_in_ipa1 = files1.difference(files2) + only_in_ipa2 = files2.difference(files1) + if len(only_in_ipa1) == 0 and len(only_in_ipa2) == 0: + return + if len(only_in_ipa1) != 0: + print('Files not present in ' + ipa2) + for f in only_in_ipa1: + print(' ' + f) + if len(only_in_ipa2) != 0: + print('Files not present in ' + ipa1) + for f in only_in_ipa2: + print(' ' + f) + + sys.exit(1) + + +def base_app_dir(path): + result = glob.glob(path + '/Payload/*.app') + if len(result) == 1: + return result[0] + else: + print('Could not find .app directory at ' + path + '/Payload') + sys.exit(1) + + +def diff_file(tempdir, self_base_path, path1, path2): + if is_plist(path1): + return diff_plists(path1, path2) + elif is_binary(path1): + return diff_binaries(tempdir, self_base_path, path1, path2) + elif is_xcconfig(path1): + return diff_xcconfigs(path1, path2) + else: + if filecmp.cmp(path1, path2): + return 'equal' + return 'not_equal' + + +def ipadiff(self_base_path, ipa1, ipa2): + tempdir = tempfile.mkdtemp() + + ipa1_dir = tempdir + '/ipa1' + ipa2_dir = tempdir + '/ipa2' + + ZipFile(ipa1, 'r').extractall(path=ipa1_dir) + ZipFile(ipa2, 'r').extractall(path=ipa2_dir) + (ipa1_dirs, ipa1_files) = get_file_list(base_app_dir(ipa1_dir)) + (ipa2_dirs, ipa2_files) = get_file_list(base_app_dir(ipa2_dir)) + + clean_ipa1_dirs = remove_codesign_dirs(ipa1_dirs) + clean_ipa2_dirs = remove_codesign_dirs(ipa2_dirs) + + clean_ipa1_files = remove_codesign_files(ipa1_files) + clean_ipa2_files = remove_codesign_files(ipa2_files) + + diff_dirs(ipa1, clean_ipa1_dirs, ipa2, clean_ipa2_dirs) + diff_files(ipa1, clean_ipa1_files, ipa2, clean_ipa2_files) + + clean_ipa1_files, watch_ipa1_files = remove_watch_files(clean_ipa1_files) + clean_ipa2_files, watch_ipa2_files = remove_watch_files(clean_ipa2_files) + + clean_ipa1_files, plugin_ipa1_files = remove_plugin_files(clean_ipa1_files) + clean_ipa2_files, plugin_ipa2_files = remove_plugin_files(clean_ipa2_files) + + clean_ipa1_files, asset_ipa1_files = remove_asset_files(clean_ipa1_files) + clean_ipa2_files, asset_ipa2_files = remove_asset_files(clean_ipa2_files) + + clean_ipa1_files, nib_ipa1_files = remove_nib_files(clean_ipa1_files) + clean_ipa2_files, nib_ipa2_files = remove_nib_files(clean_ipa2_files) + + different_files = [] + encrypted_files = [] + for relative_file_path in clean_ipa1_files: + file_result = diff_file(tempdir, self_base_path, base_app_dir(ipa1_dir) + '/' + relative_file_path, base_app_dir(ipa2_dir) + '/' + relative_file_path) + if file_result == 'equal': + pass + elif file_result == 'binary_encrypted': + encrypted_files.append(relative_file_path) + else: + different_files.append(relative_file_path) + + if len(different_files) != 0: + print('Different files in ' + ipa1 + ' and ' + ipa2) + for relative_file_path in different_files: + print(' ' + relative_file_path) + else: + if len(encrypted_files) != 0 or len(watch_ipa1_files) != 0 or len(plugin_ipa1_files) != 0: + print('IPAs are equal, except for the files that can\'t currently be checked:') + else: + print('IPAs are equal') + + if len(encrypted_files) != 0: + print(' Excluded files that couldn\'t be checked due to being encrypted:') + for relative_file_path in encrypted_files: + print(' ' + relative_file_path) + if len(watch_ipa1_files) != 0: + print(' IPAs contain Watch directory with a Watch app which currently can\'t be checked.') + if len(plugin_ipa1_files) != 0: + print(' IPAs contain PlugIns directory with app extensions. Extensions can\'t currently be checked.') + if len(asset_ipa1_files) != 0: + print(' IPAs contain .car (Asset Catalog) files that are compiled by the App Store and can\'t currently be checked:') + for relative_file_path in asset_ipa1_files: + print(' ' + relative_file_path) + if len(nib_ipa1_files) != 0: + print(' IPAs contain .nib (compiled Interface Builder) files that are compiled by the App Store and can\'t currently be checked:') + for relative_file_path in nib_ipa1_files: + print(' ' + relative_file_path) + + +if len(sys.argv) != 3: + print('Usage: ipadiff ipa1 ipa2') + sys.exit(1) + +ipadiff(os.path.dirname(sys.argv[0]), sys.argv[1], sys.argv[2]) diff --git a/tools/main.cpp b/tools/main.cpp new file mode 100644 index 0000000000..99c513eda5 --- /dev/null +++ b/tools/main.cpp @@ -0,0 +1,262 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +static uint32_t funcSwap32(uint32_t input) { + return OSSwapBigToHostInt32(input); +} + +static uint32_t funcNoSwap32(uint32_t input) { + return OSSwapLittleToHostInt32(input); +} + +static bool cleanArch(std::vector &archData, bool &isEncrypted) { + uint32_t (*swap32)(uint32_t) = funcNoSwap32; + + uint32_t offset = 0; + + const struct mach_header* header = (struct mach_header*)(archData.data() + offset); + + switch (header->magic) { + case MH_CIGAM: + swap32 = funcSwap32; + case MH_MAGIC: + offset += sizeof(struct mach_header); + break; + case MH_CIGAM_64: + swap32 = funcSwap32; + case MH_MAGIC_64: + offset += sizeof(struct mach_header_64); + break; + default: + return nullptr; + } + + uint32_t commandCount = swap32(header->ncmds); + + for (uint32_t i = 0; i < commandCount; i++) { + const struct load_command* loadCommand = (const struct load_command*)(archData.data() + offset); + uint32_t commandSize = swap32(loadCommand->cmdsize); + + uint32_t commandType = swap32(loadCommand->cmd); + if (commandType == LC_CODE_SIGNATURE) { + const struct linkedit_data_command *dataCommand = (const struct linkedit_data_command *)(archData.data() + offset); + uint32_t dataOffset = swap32(dataCommand->dataoff); + uint32_t dataSize = swap32(dataCommand->datasize); + + // account for different signature size + memset(archData.data() + offset + offsetof(linkedit_data_command, datasize), 0, sizeof(uint32_t)); + + // remove signature + archData.erase(archData.begin() + dataOffset, archData.begin() + dataOffset + dataSize); + } else if (commandType == LC_SEGMENT_64) { + const struct segment_command_64 *segmentCommand = (const struct segment_command_64 *)(archData.data() + offset); + std::string segmentName = std::string(segmentCommand->segname); + if (segmentName == "__LINKEDIT") { + // account for different signature size + memset(archData.data() + offset + offsetof(segment_command_64, vmsize), 0, sizeof(uint32_t)); + // account for different file size because of signatures + memset(archData.data() + offset + offsetof(segment_command_64, filesize), 0, sizeof(uint32_t)); + } + } else if (commandType == LC_ID_DYLIB) { + // account for dylib timestamp + memset(archData.data() + offset + offsetof(dylib_command, dylib) + offsetof(struct dylib, timestamp), 0, sizeof(uint32_t)); + } else if (commandType == LC_UUID) { + // account for dylib uuid + memset(archData.data() + offset + offsetof(uuid_command, uuid), 0, 16); + } else if (commandType == LC_ENCRYPTION_INFO_64) { + const struct encryption_info_command_64 *encryptionInfoCommand = (const struct encryption_info_command_64 *)(archData.data() + offset); + if (encryptionInfoCommand->cryptid != 0) { + isEncrypted = true; + } + } + + offset += commandSize; + } + + return true; +} + +static std::vector parseFat(std::vector const &fileData) { + size_t offset = 0; + + const struct fat_header *fatHeader = (const struct fat_header *)fileData.data(); + offset += sizeof(*fatHeader); + + size_t initialOffset = offset; + + uint32_t archCount = OSSwapBigToHostInt32(fatHeader->nfat_arch); + + for (uint32_t i = 0; i < archCount; i++) { + const struct fat_arch *arch = (const struct fat_arch *)(fileData.data() + offset); + offset += sizeof(*arch); + + uint32_t archOffset = OSSwapBigToHostInt32(arch->offset); + uint32_t archSize = OSSwapBigToHostInt32(arch->size); + cpu_type_t cputype = OSSwapBigToHostInt32(arch->cputype); + + if (cputype == CPU_TYPE_ARM64) { + std::vector archData; + archData.resize(archSize); + memcpy(archData.data(), fileData.data() + archOffset, archSize); + return archData; + } + } + + offset = initialOffset; + + for (uint32_t i = 0; i < archCount; i++) { + const struct fat_arch *arch = (const struct fat_arch *)(fileData.data() + offset); + offset += sizeof(*arch); + + uint32_t archOffset = OSSwapBigToHostInt32(arch->offset); + uint32_t archSize = OSSwapBigToHostInt32(arch->size); + cpu_type_t cputype = OSSwapBigToHostInt32(arch->cputype); + cpu_type_t cpusubtype = OSSwapBigToHostInt32(arch->cpusubtype); + + if (cputype == CPU_TYPE_ARM && cpusubtype == CPU_SUBTYPE_ARM_V7K) { + std::vector archData; + archData.resize(archSize); + memcpy(archData.data(), fileData.data() + archOffset, archSize); + return archData; + } + } + + return std::vector(); +} + +static std::vector parseMachO(std::vector const &fileData) { + const uint32_t *magic = (const uint32_t *)fileData.data(); + + if (*magic == FAT_CIGAM || *magic == FAT_MAGIC) { + return parseFat(fileData); + } else { + return fileData; + } +} + +static std::vector readFile(std::string const &file) { + int fd = open(file.c_str(), O_RDONLY); + + if (fd == -1) { + return std::vector(); + } + + struct stat st; + fstat(fd, &st); + + std::vector fileData; + fileData.resize((size_t)st.st_size); + read(fd, fileData.data(), (size_t)st.st_size); + close(fd); + + return fileData; +} + +static void writeDataToFile(std::vector const &data, std::string const &path) { + int fd = open(path.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + if (fd == -1) { + return; + } + + write(fd, data.data(), data.size()); + + close(fd); +} + +static std::vector stripSwiftSymbols(std::string const &file) { + std::string command; + command += "xcrun strip -ST -o /dev/stdout \""; + command += file; + command += "\" 2> /dev/null"; + + uint8_t buffer[128]; + std::vector result; + FILE *pipe = popen(command.c_str(), "r"); + if (!pipe) { + throw std::runtime_error("popen() failed!"); + } + while (true) { + size_t readBytes = fread(buffer, 1, 128, pipe); + if (readBytes <= 0) { + break; + } + result.insert(result.end(), buffer, buffer + readBytes); + } + pclose(pipe); + + return result; +} + +static bool endsWith(std::string const &mainStr, std::string const &toMatch) { + if(mainStr.size() >= toMatch.size() && mainStr.compare(mainStr.size() - toMatch.size(), toMatch.size(), toMatch) == 0) { + return true; + } else { + return false; + } +} + +int main(int argc, const char *argv[]) { + if (argc != 3) { + printf("Usage: machofilediff file1 file2\n"); + return 1; + } + + std::string file1 = argv[1]; + std::string file2 = argv[2]; + + std::vector fileData1; + if (endsWith(file1, ".dylib")) { + fileData1 = stripSwiftSymbols(file1); + } else { + fileData1 = readFile(file1); + } + + std::vector fileData2; + if (endsWith(file2, ".dylib")) { + fileData2 = stripSwiftSymbols(file2); + } else { + fileData2 = readFile(file2); + } + + std::vector arch1 = parseMachO(fileData1); + if (arch1.size() == 0) { + printf("Couldn't parse %s\n", file1.c_str()); + return 1; + } + + std::vector arch2 = parseMachO(fileData2); + if (arch2.size() == 0) { + printf("Couldn't parse %s\n", file2.c_str()); + return 1; + } + + bool arch1Encrypted = false; + bool arch2Encrypted = false; + cleanArch(arch1, arch1Encrypted); + cleanArch(arch2, arch2Encrypted); + + if (arch1 == arch2) { + printf("Equal\n"); + return 0; + } else { + if (arch1Encrypted || arch2Encrypted) { + printf("Encrypted\n"); + } else { + printf("Not Equal\n"); + } + + return 1; + } + + return 0; +}