Added scripts for build verification

This commit is contained in:
Peter 2019-06-24 21:28:09 +03:00
parent 1f36b62a7e
commit 3c844dbeb5
5 changed files with 602 additions and 8 deletions

View File

@ -25,6 +25,15 @@ else
exit 1
fi
COMMIT_ID=$(git rev-parse HEAD)
if [ -z "$2" ]; then
COMMIT_COUNT=$(git rev-list --count HEAD)
COMMIT_COUNT="$(($COMMIT_COUNT+1000))"
BUILD_NUMBER="$COMMIT_COUNT"
else
BUILD_NUMBER="$2"
fi
BASE_DIR=$(pwd)
if [ "$BUILD_CONFIGURATION" == "hockeyapp" ] || [ "$BUILD_CONFIGURATION" == "appstore" ]; then
@ -87,7 +96,7 @@ else
fi
scp -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr "$BUILDBOX_DIR/guest-build-telegram.sh" "$BUILDBOX_DIR/transient-data/source.tar" telegram@"$VM_IP":
ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null telegram@"$VM_IP" -o ServerAliveInterval=60 -t "export TELEGRAM_BUILD_APPSTORE_PASSWORD=\"$TELEGRAM_BUILD_APPSTORE_PASSWORD\"; export TELEGRAM_BUILD_APPSTORE_TEAM_NAME=\"$TELEGRAM_BUILD_APPSTORE_TEAM_NAME\"; bash -l guest-build-telegram.sh $BUILD_CONFIGURATION" || true
ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null telegram@"$VM_IP" -o ServerAliveInterval=60 -t "export TELEGRAM_BUILD_APPSTORE_PASSWORD=\"$TELEGRAM_BUILD_APPSTORE_PASSWORD\"; export TELEGRAM_BUILD_APPSTORE_TEAM_NAME=\"$TELEGRAM_BUILD_APPSTORE_TEAM_NAME\"; export BUILD_NUMBER=\"$BUILD_NUMBER\"; export COMMIT_ID=\"$COMMIT_ID\"; bash -l guest-build-telegram.sh $BUILD_CONFIGURATION" || true
if [ "$BUILD_CONFIGURATION" == "appstore" ]; then
ARCHIVE_PATH="$HOME/telegram-builds-archive"

View File

@ -1,5 +1,15 @@
#!/bin/sh
if [ -z "BUILD_NUMBER" ]; then
echo "BUILD_NUMBER is not set"
exit 1
fi
if [ -z "COMMIT_ID" ]; then
echo "COMMIT_ID is not set"
exit 1
fi
if [ "$1" == "hockeyapp" ]; then
FASTLANE_BUILD_CONFIGURATION="internalhockeyapp"
CERTS_PATH="codesigning_data/certs"
@ -68,5 +78,5 @@ else
tar -xf "source.tar"
cd "$SOURCE_PATH"
FASTLANE_PASSWORD="$FASTLANE_PASSWORD" FASTLANE_ITC_TEAM_NAME="$FASTLANE_ITC_TEAM_NAME" fastlane "$FASTLANE_BUILD_CONFIGURATION"
FASTLANE_PASSWORD="$FASTLANE_PASSWORD" FASTLANE_ITC_TEAM_NAME="$FASTLANE_ITC_TEAM_NAME" fastlane "$FASTLANE_BUILD_CONFIGURATION" build_number:"$BUILD_NUMBER" commit_hash:"$COMMIT_ID"
fi

View File

@ -16,6 +16,7 @@ app_identifier_llc = [
signing_identity_llc = "iPhone Distribution: Digital Fortress LLC (C67CF9S4VU)"
lane :do_build_app do |options|
puts("Building with build number: " + options[:build_number] + ", commit id: " + options[:commit_id])
gym(
workspace: "Telegram-iOS.xcworkspace",
configuration: options[:configuration],
@ -39,16 +40,12 @@ lane :do_build_app do |options|
end
lane :build_for_appstore do |options|
commit = last_git_commit
commit_count = sh("git rev-list --count HEAD")
commit_count_int = commit_count.to_i + 1000
commit_count_string = commit_count_int.to_s
do_build_app(
configuration: "ReleaseAppStoreLLC",
scheme: "Telegram-iOS-AppStoreLLC",
export_method: "app-store",
build_number: commit_count_string,
commit_id: commit[:commit_hash],
build_number: options[:build_number],
commit_id: options[:commit_hash],
signingCertificate: signing_identity_llc,
provisioningProfiles: {
base_app_identifier_llc => "match AppStore " + base_app_identifier_llc,

316
tools/ipadiff.py Normal file
View File

@ -0,0 +1,316 @@
import sys
import os
import glob
import tempfile
import re
import filecmp
import subprocess
from zipfile import ZipFile
def get_file_list(dir):
result_files = []
result_dirs = []
for root, dirs, files in os.walk(dir, topdown=False):
for name in files:
result_files.append(os.path.relpath(os.path.join(root, name), dir))
for name in dirs:
result_dirs.append(os.path.relpath(os.path.join(root, name), dir))
return set(result_dirs), set(result_files)
def remove_codesign_dirs(dirs):
result = set()
for dir in dirs:
if dir == 'SC_Info':
continue
if re.match('Watch/.*\\.appex/SC_Info', dir):
continue
if re.match('PlugIns/.*\\.appex/SC_Info', dir):
continue
if re.match('Frameworks/.*\\.framework/SC_Info', dir):
continue
result.add(dir)
return result
def remove_codesign_files(files):
result = set()
for f in files:
if f == 'embedded.mobileprovision':
continue
if re.match('.*/.*\\.appex/embedded.mobileprovision', f):
continue
if f == '_CodeSignature/CodeResources':
continue
if f == 'CrackerXI':
continue
if re.match('Watch/.*\\.app/embedded.mobileprovision', f):
continue
if re.match('PlugIns/.*\\.appex/_CodeSignature/CodeResources', f):
continue
if re.match('Frameworks/.*\\.framework/_CodeSignature/CodeResources', f):
continue
if f == 'Frameworks/ModernProto.framework/ModernProto':
continue
result.add(f)
return result
def remove_watch_files(files):
result = set()
excluded = set()
for f in files:
if re.match('Watch/.*', f):
excluded.add(f)
else:
result.add(f)
return (result, excluded)
def remove_plugin_files(files):
result = set()
excluded = set()
for f in files:
if False and re.match('PlugIns/.*', f):
excluded.add(f)
else:
result.add(f)
return (result, excluded)
def remove_asset_files(files):
result = set()
excluded = set()
for f in files:
if re.match('.*\\.car', f):
excluded.add(f)
else:
result.add(f)
return (result, excluded)
def remove_nib_files(files):
result = set()
excluded = set()
for f in files:
if re.match('.*\\.nib', f):
excluded.add(f)
else:
result.add(f)
return (result, excluded)
def diff_dirs(ipa1, dir1, ipa2, dir2):
only_in_ipa1 = dir1.difference(dir2)
only_in_ipa2 = dir2.difference(dir1)
if len(only_in_ipa1) == 0 and len(only_in_ipa2) == 0:
return
print('Directory structure doesn\'t match in ' + ipa1 + ' and ' + ipa2)
if len(only_in_ipa1) != 0:
print('Directories not present in ' + ipa2)
for dir in only_in_ipa1:
print(' ' + dir)
if len(only_in_ipa2) != 0:
print('Directories not present in ' + ipa1)
for dir in only_in_ipa2:
print(' ' + dir)
sys.exit(1)
def is_binary(file):
out = os.popen('file "' + file + '"').read()
if out.find('Mach-O') == -1:
return False
return True
def is_xcconfig(file):
if re.match('.*\\.xcconfig', file):
return True
else:
return False
def diff_binaries(tempdir, self_base_path, file1, file2):
diff_app = tempdir + '/main'
if not os.path.isfile(diff_app):
if not os.path.isfile(self_base_path + '/main.cpp'):
print('Could not find ' + self_base_path + '/main.cpp')
sys.exit(1)
subprocess.call(['clang', self_base_path + '/main.cpp', '-lc++', '-o', diff_app])
if not os.path.isfile(diff_app):
print('Could not compile ' + self_base_path + '/main.cpp')
sys.exit(1)
result = os.popen(diff_app + ' ' + file1 + ' ' + file2).read().strip()
if result == 'Encrypted':
return 'binary_encrypted'
elif result == 'Equal':
return 'equal'
elif result == 'Not Equal':
return 'not_equal'
else:
print('Unexpected data from binary diff code: ' + result)
sys.exit(1)
def is_plist(file1):
if file1.find('.plist') == -1:
return False
return True
def diff_plists(file1, file2):
remove_properties = ['UISupportedDevices', 'DTAppStoreToolsBuild', 'MinimumOSVersion', 'BuildMachineOSBuild']
clean1_properties = ''
clean2_properties = ''
with open(os.devnull, 'w') as devnull:
for property in remove_properties:
if not subprocess.call(['plutil', '-extract', property, 'xml1', '-o', '-', file1], stderr=devnull, stdout=devnull):
clean1_properties += ' | plutil -remove ' + property + ' -r -o - -- -'
if not subprocess.call(['plutil', '-extract', property, 'xml1', '-o', '-', file2], stderr=devnull, stdout=devnull):
clean2_properties += ' | plutil -remove ' + property + ' -r -o - -- -'
data1 = os.popen('plutil -convert xml1 "' + file1 + '" -o -' + clean1_properties).read()
data2 = os.popen('plutil -convert xml1 "' + file2 + '" -o -' + clean2_properties).read()
if data1 == data2:
return 'equal'
else:
with open('lhs.plist', 'wb') as f:
f.write(str.encode(data1))
with open('rhs.plist', 'wb') as f:
f.write(str.encode(data2))
sys.exit(1)
return 'not_equal'
def diff_xcconfigs(file1, file2):
with open(file1, 'rb') as f:
data1 = f.read().strip()
with open(file2, 'rb') as f:
data2 = f.read().strip()
if data1 != data2:
return 'not_equal'
return 'equal'
def diff_files(ipa1, files1, ipa2, files2):
only_in_ipa1 = files1.difference(files2)
only_in_ipa2 = files2.difference(files1)
if len(only_in_ipa1) == 0 and len(only_in_ipa2) == 0:
return
if len(only_in_ipa1) != 0:
print('Files not present in ' + ipa2)
for f in only_in_ipa1:
print(' ' + f)
if len(only_in_ipa2) != 0:
print('Files not present in ' + ipa1)
for f in only_in_ipa2:
print(' ' + f)
sys.exit(1)
def base_app_dir(path):
result = glob.glob(path + '/Payload/*.app')
if len(result) == 1:
return result[0]
else:
print('Could not find .app directory at ' + path + '/Payload')
sys.exit(1)
def diff_file(tempdir, self_base_path, path1, path2):
if is_plist(path1):
return diff_plists(path1, path2)
elif is_binary(path1):
return diff_binaries(tempdir, self_base_path, path1, path2)
elif is_xcconfig(path1):
return diff_xcconfigs(path1, path2)
else:
if filecmp.cmp(path1, path2):
return 'equal'
return 'not_equal'
def ipadiff(self_base_path, ipa1, ipa2):
tempdir = tempfile.mkdtemp()
ipa1_dir = tempdir + '/ipa1'
ipa2_dir = tempdir + '/ipa2'
ZipFile(ipa1, 'r').extractall(path=ipa1_dir)
ZipFile(ipa2, 'r').extractall(path=ipa2_dir)
(ipa1_dirs, ipa1_files) = get_file_list(base_app_dir(ipa1_dir))
(ipa2_dirs, ipa2_files) = get_file_list(base_app_dir(ipa2_dir))
clean_ipa1_dirs = remove_codesign_dirs(ipa1_dirs)
clean_ipa2_dirs = remove_codesign_dirs(ipa2_dirs)
clean_ipa1_files = remove_codesign_files(ipa1_files)
clean_ipa2_files = remove_codesign_files(ipa2_files)
diff_dirs(ipa1, clean_ipa1_dirs, ipa2, clean_ipa2_dirs)
diff_files(ipa1, clean_ipa1_files, ipa2, clean_ipa2_files)
clean_ipa1_files, watch_ipa1_files = remove_watch_files(clean_ipa1_files)
clean_ipa2_files, watch_ipa2_files = remove_watch_files(clean_ipa2_files)
clean_ipa1_files, plugin_ipa1_files = remove_plugin_files(clean_ipa1_files)
clean_ipa2_files, plugin_ipa2_files = remove_plugin_files(clean_ipa2_files)
clean_ipa1_files, asset_ipa1_files = remove_asset_files(clean_ipa1_files)
clean_ipa2_files, asset_ipa2_files = remove_asset_files(clean_ipa2_files)
clean_ipa1_files, nib_ipa1_files = remove_nib_files(clean_ipa1_files)
clean_ipa2_files, nib_ipa2_files = remove_nib_files(clean_ipa2_files)
different_files = []
encrypted_files = []
for relative_file_path in clean_ipa1_files:
file_result = diff_file(tempdir, self_base_path, base_app_dir(ipa1_dir) + '/' + relative_file_path, base_app_dir(ipa2_dir) + '/' + relative_file_path)
if file_result == 'equal':
pass
elif file_result == 'binary_encrypted':
encrypted_files.append(relative_file_path)
else:
different_files.append(relative_file_path)
if len(different_files) != 0:
print('Different files in ' + ipa1 + ' and ' + ipa2)
for relative_file_path in different_files:
print(' ' + relative_file_path)
else:
if len(encrypted_files) != 0 or len(watch_ipa1_files) != 0 or len(plugin_ipa1_files) != 0:
print('IPAs are equal, except for the files that can\'t currently be checked:')
else:
print('IPAs are equal')
if len(encrypted_files) != 0:
print(' Excluded files that couldn\'t be checked due to being encrypted:')
for relative_file_path in encrypted_files:
print(' ' + relative_file_path)
if len(watch_ipa1_files) != 0:
print(' IPAs contain Watch directory with a Watch app which currently can\'t be checked.')
if len(plugin_ipa1_files) != 0:
print(' IPAs contain PlugIns directory with app extensions. Extensions can\'t currently be checked.')
if len(asset_ipa1_files) != 0:
print(' IPAs contain .car (Asset Catalog) files that are compiled by the App Store and can\'t currently be checked:')
for relative_file_path in asset_ipa1_files:
print(' ' + relative_file_path)
if len(nib_ipa1_files) != 0:
print(' IPAs contain .nib (compiled Interface Builder) files that are compiled by the App Store and can\'t currently be checked:')
for relative_file_path in nib_ipa1_files:
print(' ' + relative_file_path)
if len(sys.argv) != 3:
print('Usage: ipadiff ipa1 ipa2')
sys.exit(1)
ipadiff(os.path.dirname(sys.argv[0]), sys.argv[1], sys.argv[2])

262
tools/main.cpp Normal file
View File

@ -0,0 +1,262 @@
#include <mach-o/arch.h>
#include <mach-o/loader.h>
#include <mach-o/fat.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <stdio.h>
#include <cstdio>
#include <vector>
#include <string>
#include <array>
static uint32_t funcSwap32(uint32_t input) {
return OSSwapBigToHostInt32(input);
}
static uint32_t funcNoSwap32(uint32_t input) {
return OSSwapLittleToHostInt32(input);
}
static bool cleanArch(std::vector<uint8_t> &archData, bool &isEncrypted) {
uint32_t (*swap32)(uint32_t) = funcNoSwap32;
uint32_t offset = 0;
const struct mach_header* header = (struct mach_header*)(archData.data() + offset);
switch (header->magic) {
case MH_CIGAM:
swap32 = funcSwap32;
case MH_MAGIC:
offset += sizeof(struct mach_header);
break;
case MH_CIGAM_64:
swap32 = funcSwap32;
case MH_MAGIC_64:
offset += sizeof(struct mach_header_64);
break;
default:
return nullptr;
}
uint32_t commandCount = swap32(header->ncmds);
for (uint32_t i = 0; i < commandCount; i++) {
const struct load_command* loadCommand = (const struct load_command*)(archData.data() + offset);
uint32_t commandSize = swap32(loadCommand->cmdsize);
uint32_t commandType = swap32(loadCommand->cmd);
if (commandType == LC_CODE_SIGNATURE) {
const struct linkedit_data_command *dataCommand = (const struct linkedit_data_command *)(archData.data() + offset);
uint32_t dataOffset = swap32(dataCommand->dataoff);
uint32_t dataSize = swap32(dataCommand->datasize);
// account for different signature size
memset(archData.data() + offset + offsetof(linkedit_data_command, datasize), 0, sizeof(uint32_t));
// remove signature
archData.erase(archData.begin() + dataOffset, archData.begin() + dataOffset + dataSize);
} else if (commandType == LC_SEGMENT_64) {
const struct segment_command_64 *segmentCommand = (const struct segment_command_64 *)(archData.data() + offset);
std::string segmentName = std::string(segmentCommand->segname);
if (segmentName == "__LINKEDIT") {
// account for different signature size
memset(archData.data() + offset + offsetof(segment_command_64, vmsize), 0, sizeof(uint32_t));
// account for different file size because of signatures
memset(archData.data() + offset + offsetof(segment_command_64, filesize), 0, sizeof(uint32_t));
}
} else if (commandType == LC_ID_DYLIB) {
// account for dylib timestamp
memset(archData.data() + offset + offsetof(dylib_command, dylib) + offsetof(struct dylib, timestamp), 0, sizeof(uint32_t));
} else if (commandType == LC_UUID) {
// account for dylib uuid
memset(archData.data() + offset + offsetof(uuid_command, uuid), 0, 16);
} else if (commandType == LC_ENCRYPTION_INFO_64) {
const struct encryption_info_command_64 *encryptionInfoCommand = (const struct encryption_info_command_64 *)(archData.data() + offset);
if (encryptionInfoCommand->cryptid != 0) {
isEncrypted = true;
}
}
offset += commandSize;
}
return true;
}
static std::vector<uint8_t> parseFat(std::vector<uint8_t> const &fileData) {
size_t offset = 0;
const struct fat_header *fatHeader = (const struct fat_header *)fileData.data();
offset += sizeof(*fatHeader);
size_t initialOffset = offset;
uint32_t archCount = OSSwapBigToHostInt32(fatHeader->nfat_arch);
for (uint32_t i = 0; i < archCount; i++) {
const struct fat_arch *arch = (const struct fat_arch *)(fileData.data() + offset);
offset += sizeof(*arch);
uint32_t archOffset = OSSwapBigToHostInt32(arch->offset);
uint32_t archSize = OSSwapBigToHostInt32(arch->size);
cpu_type_t cputype = OSSwapBigToHostInt32(arch->cputype);
if (cputype == CPU_TYPE_ARM64) {
std::vector<uint8_t> archData;
archData.resize(archSize);
memcpy(archData.data(), fileData.data() + archOffset, archSize);
return archData;
}
}
offset = initialOffset;
for (uint32_t i = 0; i < archCount; i++) {
const struct fat_arch *arch = (const struct fat_arch *)(fileData.data() + offset);
offset += sizeof(*arch);
uint32_t archOffset = OSSwapBigToHostInt32(arch->offset);
uint32_t archSize = OSSwapBigToHostInt32(arch->size);
cpu_type_t cputype = OSSwapBigToHostInt32(arch->cputype);
cpu_type_t cpusubtype = OSSwapBigToHostInt32(arch->cpusubtype);
if (cputype == CPU_TYPE_ARM && cpusubtype == CPU_SUBTYPE_ARM_V7K) {
std::vector<uint8_t> archData;
archData.resize(archSize);
memcpy(archData.data(), fileData.data() + archOffset, archSize);
return archData;
}
}
return std::vector<uint8_t>();
}
static std::vector<uint8_t> parseMachO(std::vector<uint8_t> const &fileData) {
const uint32_t *magic = (const uint32_t *)fileData.data();
if (*magic == FAT_CIGAM || *magic == FAT_MAGIC) {
return parseFat(fileData);
} else {
return fileData;
}
}
static std::vector<uint8_t> readFile(std::string const &file) {
int fd = open(file.c_str(), O_RDONLY);
if (fd == -1) {
return std::vector<uint8_t>();
}
struct stat st;
fstat(fd, &st);
std::vector<uint8_t> fileData;
fileData.resize((size_t)st.st_size);
read(fd, fileData.data(), (size_t)st.st_size);
close(fd);
return fileData;
}
static void writeDataToFile(std::vector<uint8_t> const &data, std::string const &path) {
int fd = open(path.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
if (fd == -1) {
return;
}
write(fd, data.data(), data.size());
close(fd);
}
static std::vector<uint8_t> stripSwiftSymbols(std::string const &file) {
std::string command;
command += "xcrun strip -ST -o /dev/stdout \"";
command += file;
command += "\" 2> /dev/null";
uint8_t buffer[128];
std::vector<uint8_t> result;
FILE *pipe = popen(command.c_str(), "r");
if (!pipe) {
throw std::runtime_error("popen() failed!");
}
while (true) {
size_t readBytes = fread(buffer, 1, 128, pipe);
if (readBytes <= 0) {
break;
}
result.insert(result.end(), buffer, buffer + readBytes);
}
pclose(pipe);
return result;
}
static bool endsWith(std::string const &mainStr, std::string const &toMatch) {
if(mainStr.size() >= toMatch.size() && mainStr.compare(mainStr.size() - toMatch.size(), toMatch.size(), toMatch) == 0) {
return true;
} else {
return false;
}
}
int main(int argc, const char *argv[]) {
if (argc != 3) {
printf("Usage: machofilediff file1 file2\n");
return 1;
}
std::string file1 = argv[1];
std::string file2 = argv[2];
std::vector<uint8_t> fileData1;
if (endsWith(file1, ".dylib")) {
fileData1 = stripSwiftSymbols(file1);
} else {
fileData1 = readFile(file1);
}
std::vector<uint8_t> fileData2;
if (endsWith(file2, ".dylib")) {
fileData2 = stripSwiftSymbols(file2);
} else {
fileData2 = readFile(file2);
}
std::vector<uint8_t> arch1 = parseMachO(fileData1);
if (arch1.size() == 0) {
printf("Couldn't parse %s\n", file1.c_str());
return 1;
}
std::vector<uint8_t> arch2 = parseMachO(fileData2);
if (arch2.size() == 0) {
printf("Couldn't parse %s\n", file2.c_str());
return 1;
}
bool arch1Encrypted = false;
bool arch2Encrypted = false;
cleanArch(arch1, arch1Encrypted);
cleanArch(arch2, arch2Encrypted);
if (arch1 == arch2) {
printf("Equal\n");
return 0;
} else {
if (arch1Encrypted || arch2Encrypted) {
printf("Encrypted\n");
} else {
printf("Not Equal\n");
}
return 1;
}
return 0;
}