Refactor string generation

This commit is contained in:
Ali
2021-07-20 11:57:50 +02:00
parent bbd096d88b
commit 7bed1728bb
205 changed files with 15848 additions and 33988 deletions

View File

@@ -4,9 +4,9 @@
import argparse
import sys
import os
import re
import codecs
import struct
def _unescape_key(s):
@@ -142,6 +142,18 @@ def write_string(file, string: str):
file.write((string + '\n').encode('utf-8'))
def write_bin_uint32(file, value: int):
file.write(struct.pack('I', value))
def write_bin_uint8(file, value: int):
file.write(struct.pack('B', value))
def write_bin_string(file, value: str):
file.write(value.encode('utf-8'))
class IndexCounter:
def __init__(self):
self.dictionary = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
@@ -188,71 +200,426 @@ def sanitize_entry_identifer(string: str) -> str:
return string.replace('.', '_')
def generate(header_path: str, implementation_path: str, entries: [Entry]):
test_path = '/Users/ali/build/telegram/telegram-ios/submodules/PresentationStrings/main.m'
def generate(header_path: str, implementation_path: str, data_path: str, entries: [Entry]):
print('Generating strings into:\n{}\n{}'.format(header_path, implementation_path))
with open(header_path, 'wb') as header_file, open(implementation_path, 'wb') as source_file, open(test_path, 'wb') as test_file:
write_string(test_file, '#import <Foundation/Foundation.h>')
write_string(test_file, '#import <PresentationStrings/PresentationStrings.h>')
write_string(test_file, '')
write_string(test_file, 'int main(int argc, const char **argv) {')
write_string(test_file, 'PresentationStrings *strings = [[PresentationStrings alloc] init];')
with open(header_path, 'wb') as header_file, open(implementation_path, 'wb') as source_file,\
open(data_path, 'wb') as data_file:
write_string(header_file, '#import <Foundation/Foundation.h>')
write_string(header_file, '')
write_string(header_file, '@interface PresentationStrings : NSObject')
write_string(header_file, '')
write_string(header_file, '@end')
formatted_accessors = ''
max_format_argument_count = 0
for entry in entries:
num_arguments = len(entry.positional_arguments)
if num_arguments > max_format_argument_count:
max_format_argument_count = num_arguments
if max_format_argument_count != 0:
for num_arguments in range(1, max_format_argument_count + 1):
arguments_string = ''
arguments_array = ''
for i in range(0, num_arguments):
arguments_string += ', id arg{}'.format(i)
if i != 0:
arguments_array += ', '
arguments_array += 'arg{}'.format(i)
formatted_accessors += '''
static _FormattedString * _Nonnull getFormatted{num_arguments}(_PresentationStrings * _Nonnull strings,
uint32_t keyId{arguments_string}) {{
NSString *formatString = getSingle(strings, strings->_idToKey[@(keyId)]);
NSArray<_FormattedStringRange *> *argumentRanges = extractArgumentRanges(formatString);
return formatWithArgumentRanges(formatString, argumentRanges, @[{arguments_array}]);
}}
'''.format(num_arguments=num_arguments, arguments_string=arguments_string, arguments_array=arguments_array)
write_string(source_file, '#import <PresentationStrings/PresentationStrings.h>')
write_string(source_file, '')
write_string(source_file, '@implementation PresentationStrings')
write_string(source_file, '@end')
write_string(source_file, '')
write_string(header_file, '''// Automatically-generated file, do not edit
#import <Foundation/Foundation.h>
@interface _FormattedStringRange : NSObject
@property (nonatomic, readonly) NSInteger index;
@property (nonatomic, readonly) NSRange range;
- (instancetype _Nonnull)initWithIndex:(NSInteger)index range:(NSRange)range;
@end
@interface _FormattedString : NSObject
@property (nonatomic, strong, readonly) NSString * _Nonnull string;
@property (nonatomic, strong, readonly) NSArray<_FormattedStringRange *> * _Nonnull ranges;
- (instancetype _Nonnull)initWithString:(NSString * _Nonnull)string
ranges:(NSArray<_FormattedStringRange *> * _Nonnull)ranges;
@end
@interface _PresentationStringsComponent : NSObject
@property (nonatomic, strong, readonly) NSString * _Nonnull languageCode;
@property (nonatomic, strong, readonly) NSString * _Nonnull localizedName;
@property (nonatomic, strong, readonly) NSString * _Nullable pluralizationRulesCode;
@property (nonatomic, strong, readonly) NSDictionary<NSString *, NSString *> * _Nonnull dict;
- (instancetype _Nonnull)initWithLanguageCode:(NSString * _Nonnull)languageCode
localizedName:(NSString * _Nonnull)localizedName
pluralizationRulesCode:(NSString * _Nullable)pluralizationRulesCode
dict:(NSDictionary<NSString *, NSString *> * _Nonnull)dict;
@end
@interface _PresentationStrings : NSObject
@property (nonatomic, readonly) uint32_t lc;
@property (nonatomic, strong, readonly) _PresentationStringsComponent * _Nonnull primaryComponent;
@property (nonatomic, strong, readonly) _PresentationStringsComponent * _Nullable secondaryComponent;
@property (nonatomic, strong, readonly) NSString * _Nonnull baseLanguageCode;
@property (nonatomic, strong, readonly) NSString * _Nonnull groupingSeparator;
- (instancetype _Nonnull)initWithPrimaryComponent:(_PresentationStringsComponent * _Nonnull)primaryComponent
secondaryComponent:(_PresentationStringsComponent * _Nullable)secondaryComponent
groupingSeparator:(NSString * _Nullable)groupingSeparator;
@end
''')
write_string(source_file, '''// Automatically-generated file, do not edit
#import <PresentationStrings/PresentationStrings.h>
#import <PresentationStrings/StringPluralization.h>
@implementation _FormattedStringRange
- (instancetype _Nonnull)initWithIndex:(NSInteger)index range:(NSRange)range {
self = [super init];
if (self != nil) {
_index = index;
_range = range;
}
return self;
}
@end
@implementation _FormattedString
- (instancetype _Nonnull)initWithString:(NSString * _Nonnull)string
ranges:(NSArray<_FormattedStringRange *> * _Nonnull)ranges {
self = [super init];
if (self != nil) {
_string = string;
_ranges = ranges;
}
return self;
}
@end
@implementation _PresentationStringsComponent
- (instancetype _Nonnull)initWithLanguageCode:(NSString * _Nonnull)languageCode
localizedName:(NSString * _Nonnull)localizedName
pluralizationRulesCode:(NSString * _Nullable)pluralizationRulesCode
dict:(NSDictionary<NSString *, NSString *> * _Nonnull)dict {
self = [super init];
if (self != nil) {
_languageCode = languageCode;
_localizedName = localizedName;
_pluralizationRulesCode = pluralizationRulesCode;
_dict = dict;
}
return self;
}
@end
@interface _PresentationStrings () {
@public
NSDictionary<NSNumber *, NSString *> *_idToKey;
}
@end
static NSArray<_FormattedStringRange *> * _Nonnull extractArgumentRanges(NSString * _Nonnull string) {
static NSRegularExpression *argumentRegex = nil;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
argumentRegex = [NSRegularExpression regularExpressionWithPattern:@"%(((\\\\d+)\\\\$)?)([@df])"
options:0 error:nil];
});
NSMutableArray<_FormattedStringRange *> *result = [[NSMutableArray alloc] init];
NSArray<NSTextCheckingResult *> *matches = [argumentRegex matchesInString:string
options:0 range:NSMakeRange(0, string.length)];
int index = 0;
for (NSTextCheckingResult *match in matches) {
int currentIndex = index;
NSRange matchRange = [match rangeAtIndex:3];
if (matchRange.location != NSNotFound) {
currentIndex = [[string substringWithRange:matchRange] intValue] - 1;
}
[result addObject:[[_FormattedStringRange alloc] initWithIndex:currentIndex range:[match rangeAtIndex:0]]];
index += 1;
}
//sort?
return result;
}
static _FormattedString * _Nonnull formatWithArgumentRanges(
NSString * _Nonnull string,
NSArray<_FormattedStringRange *> * _Nonnull ranges,
NSArray<NSString *> * _Nonnull arguments
) {
NSMutableArray<_FormattedStringRange *> *resultingRanges = [[NSMutableArray alloc] init];
NSMutableString *result = [[NSMutableString alloc] init];
NSUInteger currentLocation = 0;
for (_FormattedStringRange *range in ranges) {
if (currentLocation < range.range.location) {
[result appendString:[string substringWithRange:
NSMakeRange(currentLocation, range.range.location - currentLocation)]];
}
[resultingRanges addObject:[[_FormattedStringRange alloc] initWithIndex:range.index
range:NSMakeRange(result.length, arguments[range.index].length)]];
[result appendString:arguments[range.index]];
currentLocation = range.range.location + range.range.length;
}
if (currentLocation != string.length) {
[result appendString:[string substringWithRange:NSMakeRange(currentLocation, string.length - currentLocation)]];
}
return [[_FormattedString alloc] initWithString:result ranges:resultingRanges];
}
static NSString * _Nonnull getPluralizationSuffix(_PresentationStrings * _Nonnull strings, int32_t value) {
StringPluralizationForm pluralizationForm = getStringPluralizationForm(strings.lc, value);
switch (pluralizationForm) {
case StringPluralizationFormZero: {
return @"_0";
}
case StringPluralizationFormOne: {
return @"_1";
}
case StringPluralizationFormTwo: {
return @"_2";
}
case StringPluralizationFormFew: {
return @"_3_10";
}
case StringPluralizationFormMany: {
return @"_many";
}
default: {
return @"_any";
}
}
}
static NSString * _Nonnull getSingle(_PresentationStrings * _Nonnull strings, NSString * _Nonnull key) {
NSString *result = strings.primaryComponent.dict[key];
if (!result) {
result = strings.secondaryComponent.dict[key];
}
if (!result) {
static NSDictionary<NSString *, NSString *> *fallbackDict = nil;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
NSString *lprojPath = [[NSBundle mainBundle] pathForResource:@"en" ofType:@"lproj"];
if (!lprojPath) {
return;
}
NSBundle *bundle = [NSBundle bundleWithPath:lprojPath];
if (!bundle) {
return;
}
NSString *stringsPath = [bundle pathForResource:@"Localizable" ofType:@"strings"];
if (!stringsPath) {
return;
}
fallbackDict = [NSDictionary dictionaryWithContentsOfURL:[NSURL fileURLWithPath:stringsPath]];
});
result = fallbackDict[key];
}
if (!result) {
result = key;
}
return result;
}
static NSString * _Nonnull getSingleIndirect(_PresentationStrings * _Nonnull strings, uint32_t keyId) {
return getSingle(strings, strings->_idToKey[@(keyId)]);
}
static NSString * _Nonnull getPluralized(_PresentationStrings * _Nonnull strings, NSString * _Nonnull key,
int32_t value) {
NSString *parsedKey = [[NSString alloc] initWithFormat:@"%@%@", key, getPluralizationSuffix(strings, value)];
NSString *formatString = getSingle(strings, parsedKey);
NSString *stringValue = [[NSString alloc] initWithFormat:@"%d", (int)value];
NSString *result = [[NSString alloc] initWithFormat:formatString, stringValue];
return result;
}
static NSString * _Nonnull getPluralizedIndirect(_PresentationStrings * _Nonnull strings, uint32_t keyId,
int32_t value) {
return getPluralized(strings, strings->_idToKey[@(keyId)], value);
}''' + formatted_accessors + '''
@implementation _PresentationStrings
- (instancetype _Nonnull)initWithPrimaryComponent:(_PresentationStringsComponent * _Nonnull)primaryComponent
secondaryComponent:(_PresentationStringsComponent * _Nullable)secondaryComponent
groupingSeparator:(NSString * _Nullable)groupingSeparator {
self = [super init];
if (self != nil) {
static NSDictionary<NSNumber *, NSString *> *idToKey = nil;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
NSString *dataPath = [[NSBundle mainBundle] pathForResource:@"PresentationStrings" ofType:@"data"];
if (!dataPath) {
assert(false);
return;
}
NSData *data = [NSData dataWithContentsOfURL:[NSURL fileURLWithPath:dataPath]];
if (!data) {
assert(false);
return;
}
if (data.length < 4) {
assert(false);
return;
}
NSMutableDictionary<NSNumber *, NSString *> *result = [[NSMutableDictionary alloc] init];
uint32_t entryCount = 0;
[data getBytes:&entryCount range:NSMakeRange(0, 4)];
NSInteger offset = 4;
for (uint32_t i = 0; i < entryCount; i++) {
uint8_t stringLength = 0;
[data getBytes:&stringLength range:NSMakeRange(offset, 1)];
offset += 1;
NSData *stringData = [data subdataWithRange:NSMakeRange(offset, stringLength)];
offset += stringLength;
result[@(i)] = [[NSString alloc] initWithData:stringData encoding:NSUTF8StringEncoding];
}
idToKey = result;
});
_idToKey = idToKey;
_primaryComponent = primaryComponent;
_secondaryComponent = secondaryComponent;
_groupingSeparator = groupingSeparator;
if (secondaryComponent) {
_baseLanguageCode = secondaryComponent.languageCode;
} else {
_baseLanguageCode = primaryComponent.languageCode;
}
NSString *languageCode = nil;
if (primaryComponent.pluralizationRulesCode) {
languageCode = primaryComponent.pluralizationRulesCode;
} else {
languageCode = primaryComponent.languageCode;
}
NSString *rawCode = languageCode;
NSRange range = [languageCode rangeOfString:@"_"];
if (range.location != NSNotFound) {
rawCode = [rawCode substringWithRange:NSMakeRange(0, range.location)];
}
range = [languageCode rangeOfString:@"-"];
if (range.location != NSNotFound) {
rawCode = [rawCode substringWithRange:NSMakeRange(0, range.location)];
}
rawCode = [rawCode lowercaseString];
uint32_t lc = 0;
for (NSInteger i = 0; i < rawCode.length; i++) {
lc = (lc << 8) + (uint32_t)[rawCode characterAtIndex:i];
}
_lc = lc;
}
return self;
}
@end
''')
counter = IndexCounter()
entry_keys = []
for entry in entries:
entry_id = '_L' + counter.get_next_valid_id()
write_string(header_file, '// {}'.format(entry.name))
entry_key_id = len(entry_keys)
entry_keys.append(entry.name)
write_string(source_file, '// {}'.format(entry.name))
function_arguments = ''
format_arguments_array = ''
if entry.is_pluralized:
swift_spec = 'PresentationStrings.{}(self:_:)'.format(sanitize_entry_identifer(entry.name))
function_return_spec = 'NSString * _Nonnull'
swift_spec = '_PresentationStrings.{}(self:_:)'.format(sanitize_entry_identifer(entry.name))
function_arguments = ', int32_t value'
elif len(entry.positional_arguments) != 0:
function_return_spec = '_FormattedString * _Nonnull'
positional_arguments_spec = ''
argument_index = -1
for argument in entry.positional_arguments:
argument_index += 1
format_arguments_array += ', '
if argument.kind == 'd':
function_arguments += ', int32_t _{}'.format(argument_index)
function_arguments += ', NSInteger _{}'.format(argument_index)
format_arguments_array += '@(_{})'\
.format(argument_index)
elif argument.kind == '@':
function_arguments += ', NSString * _Nonnull _{}'.format(argument_index)
format_arguments_array += '_{}'.format(argument_index)
else:
raise Exception('Unsupported argument type {}'.format(argument.kind))
positional_arguments_spec += '_:'
swift_spec = 'PresentationStrings.{}(self:{})'.format(
swift_spec = '_PresentationStrings.{}(self:{})'.format(
sanitize_entry_identifer(entry.name), positional_arguments_spec)
else:
swift_spec = 'getter:PresentationStrings.{}(self:)'.format(sanitize_entry_identifer(entry.name))
# write_string(header_file, '__attribute__((swift_name("{}")))'
# .format(swift_spec))
write_string(header_file, 'NSString * _Nonnull {}(PresentationStrings * _Nonnull _self);'.format(
entry_id, function_arguments))
function_return_spec = 'NSString * _Nonnull'
swift_spec = 'getter:_PresentationStrings.{}(self:)'.format(sanitize_entry_identifer(entry.name))
write_string(test_file, '{}(strings);'.format(entry_id))
write_string(test_file, '{}(strings);'.format(entry_id))
write_string(test_file, '{}(strings);'.format(entry_id))
function_spec = '{} {}'.format(function_return_spec, entry_id)
function_spec += '(_PresentationStrings * _Nonnull _self{})'.format(function_arguments)
write_string(source_file, 'NSString * _Nonnull {}(PresentationStrings * _Nonnull _self) {{'.format(
entry_id, function_arguments))
write_string(source_file, ' return @"";'.format(entry_id))
write_string(source_file, '}')
write_string(source_file, ''.format(entry_id))
write_string(test_file, ' return 0;')
write_string(test_file, '}')
write_string(test_file, '')
write_string(header_file, '{function_spec} __attribute__((__swift_name__("{swift_spec}")));'.format(
function_spec=function_spec, swift_spec=swift_spec))
if entry.is_pluralized:
write_string(source_file, function_spec + ''' {{
return getPluralizedIndirect(_self, {entry_key_id}, value);
}}'''.format(key=entry.name, entry_key_id=entry_key_id))
elif len(entry.positional_arguments) != 0:
write_string(source_file, function_spec + ''' {{
return getFormatted{argument_count}(_self, {key_id}{arguments_array});
}}'''.format(key_id=entry_key_id, argument_count=len(entry.positional_arguments), arguments_array=format_arguments_array))
else:
write_string(source_file, function_spec + ''' {{
return getSingleIndirect(_self, {entry_key_id});
}}'''.format(key=entry.name, entry_key_id=entry_key_id))
write_bin_uint32(data_file, len(entry_keys))
for entry_key in entry_keys:
write_bin_uint8(data_file, len(entry_key))
write_bin_string(data_file, entry_key)
if __name__ == '__main__':
@@ -276,6 +643,12 @@ if __name__ == '__main__':
help='Path to PresentationStrings.h',
metavar='path'
)
parser.add_argument(
'--outData',
required=True,
help='Path to PresentationStrings.data',
metavar='path'
)
if len(sys.argv) < 2:
parser.print_help()
@@ -285,4 +658,5 @@ if __name__ == '__main__':
parsed_strings = parse_strings(args.source)
all_entries = parse_entries(parsed_strings)
generate(header_path=args.outHeader, implementation_path=args.outImplementation, entries=all_entries)
generate(header_path=args.outHeader, implementation_path=args.outImplementation, data_path=args.outData,
entries=all_entries)