Build system experiments

build-system/bazel-utils/spm.bzl

@@ -317,6 +317,17 @@ def _collect_spm_modules_impl(target, ctx):
                 headers.append(hdr_file.path)
     current_target_headers = depset(current_target_hdr_files)
 
+    textual_hdrs = []
+    current_target_textual_hdr_files = []
+    if "textual_hdrs" in result_attrs:
+        for textual_hdr_target in result_attrs["textual_hdrs"]:
+            textual_hdr_files = textual_hdr_target.files.to_list()
+            for f in textual_hdr_files:
+                current_target_textual_hdr_files.append(f)
+            for hdr_file in textual_hdr_files:
+                textual_hdrs.append(hdr_file.path)
+    current_target_textual_headers = depset(current_target_textual_hdr_files)
+
     module_type = result_attrs["type"]
 
     if module_type == "root":
@@ -368,7 +379,7 @@ def _collect_spm_modules_impl(target, ctx):
                 "path": module_path,
                 "defines": result_attrs["defines"],
                 "deps": dep_names,
-                "sources": sorted(sources + headers),
+                "sources": sorted(sources + headers + textual_hdrs),
                 "module_name": module_name,
                 "copts": result_attrs["copts"],
                 "cxxopts": result_attrs["cxxopts"],
@@ -401,6 +412,7 @@ def _collect_spm_modules_impl(target, ctx):
         transitive_sources_from_deps, 
         current_target_sources, 
         current_target_headers,
+        current_target_textual_headers,
     ])
     
     # Return both the SPM output files and the provider with modules data and sources

build-system/generate_spm.py

@@ -5,6 +5,7 @@ import os
 import sys
 import json
 import shutil
+import re
 
 # Read the modules JSON file
 modules_json_path = "bazel-bin/Telegram/spm_build_root_modules.json"
@@ -15,18 +16,40 @@ with open(modules_json_path, 'r') as f:
 # Clean spm-files
 spm_files_dir = "spm-files"
 if os.path.exists(spm_files_dir):
-    shutil.rmtree(spm_files_dir)
+    for item in os.listdir(spm_files_dir):
+        if item != ".build":
+            item_path = os.path.join(spm_files_dir, item)
+            if os.path.isfile(item_path):
+                os.unlink(item_path)
+            elif os.path.isdir(item_path):
+                shutil.rmtree(item_path)
 if not os.path.exists(spm_files_dir):
     os.makedirs(spm_files_dir)
 
 def escape_swift_string_literal_component(text: str) -> str:
+    # Handle -D defines that use shell-style quoting like -DPACKAGE_STRING='""'
+    # In Bazel, this gets processed by shell to become -DPACKAGE_STRING=""
+    # In SwiftPM, we need to manually do this processing
+    if text.startswith("-D") and "=" in text:
+        # Split on the first = to get key and value parts
+        define_part, value_part = text.split("=", 1)
+        
+        # Check if value is wrapped in single quotes (shell-style escaping)
+        if value_part.startswith("'") and value_part.endswith("'") and len(value_part) >= 2:
+            # Remove the outer single quotes
+            inner_value = value_part[1:-1]
+            # Escape the inner value for Swift string literal
+            escaped_inner = inner_value.replace('\\', '\\\\').replace('"', '\\"')
+            return f"{define_part}={escaped_inner}"
+    
+    # For non-define flags or defines without shell quoting, just escape for Swift string literal
     return text.replace('\\', '\\\\').replace('"', '\\"')
 
 parsed_modules = {}
 for name, module in sorted(modules.items()):
     is_empty = False
     all_source_files = []
-    for source in module.get("hdrs", []) + module["sources"]:
+    for source in module.get("hdrs", []) + module.get("textual_hdrs", []) + module["sources"]:
         if source.endswith(('.a')):
             continue
         all_source_files.append(source)
@@ -69,18 +92,11 @@ for name, module in sorted(modules.items()):
         combined_lines.append("        .target(")
         combined_lines.append("            name: \"%s\"," % name)
 
-        linked_directory = None
+        # Always create a symlinked directory for every module
-        has_non_linked_sources = False
+
-        for source in module["sources"] + module.get("hdrs", []):
+        relative_module_path = module["path"]
-            if source.startswith("bazel-out/"):
+        module_directory = spm_files_dir + "/" + relative_module_path
-                linked_directory = "spm-files/" + name
+        os.makedirs(module_directory, exist_ok=True)
-            else:
-                has_non_linked_sources = True
-        if linked_directory and has_non_linked_sources:
-            print("Module {} has both regular and generated sources".format(name))
-            sys.exit(1)
-        if linked_directory:
-            os.makedirs(linked_directory)
         
         combined_lines.append("            dependencies: [")
         for dep in module["deps"]:
@@ -88,91 +104,60 @@ for name, module in sorted(modules.items()):
                 combined_lines.append("                .target(name: \"%s\")," % dep)
         combined_lines.append("            ],")
 
-        if linked_directory:
+        # All modules now use the symlinked directory path
-            combined_lines.append("            path: \"%s\"," % linked_directory)
+        combined_lines.append("            path: \"%s\"," % relative_module_path)
-        else:
-            combined_lines.append("            path: \"%s\"," % module["path"])
 
+        # Since we control the entire directory structure, we don't need exclude logic
         combined_lines.append("            exclude: [")
-        exclude_files_and_dirs = []
-        if not linked_directory:
-            for root, dirs, files in os.walk(module["path"]):
-                rel_path = os.path.relpath(root, module["path"])
-                if rel_path == ".":
-                    rel_path = ""
-                else:
-                    rel_path += "/"
-                
-                # Add directories that should be excluded
-                for d in dirs:
-                    dir_path = os.path.join(rel_path, d)
-                    if any(component.startswith('.') for component in dir_path.split('/')):
-                        continue
-                    # Check if any source file is under this directory
-                    has_source = False
-                    for source in module["sources"] + module.get("hdrs", []):
-                        rel_source = source[len(module["path"]) + 1:]
-                        if rel_source.startswith(dir_path + "/"):
-                            has_source = True
-                            break
-                    if not has_source:
-                        exclude_files_and_dirs.append(dir_path)
-                
-                # Add files that should be excluded
-                for f in files:
-                    file_path = os.path.join(rel_path, f)
-                    if any(component.startswith('.') for component in file_path.split('/')):
-                        continue
-                    if file_path not in [source[len(module["path"]) + 1:] for source in module["sources"] + module.get("hdrs", [])]:
-                        exclude_files_and_dirs.append(file_path)
-        for item in exclude_files_and_dirs:
-            combined_lines.append("                \"%s\"," % item)
         combined_lines.append("            ],")
         
         combined_lines.append("            sources: [")
-        for source in module["sources"] + module.get("hdrs", []):
+        for source in module["sources"] + module.get("hdrs", []) + module.get("textual_hdrs", []):
-            linked_source_file_names = []
+            # Process all sources (both regular and generated) with symlinks
-            if not source.startswith(module["path"]):
+            if source.startswith("bazel-out/"):
-                if source.startswith("bazel-out/"):
+                # Generated file - extract relative path within module
-                    if not linked_directory:
+                if module["path"] in source:
-                        print("Source {} is a generated file, but module {} has no linked directory".format(source, name))
+                    source_file_name = source[source.index(module["path"]) + len(module["path"]) + 1:]
-                        sys.exit(1)
-                    if module["path"] in source:
-                        source_file_name = source[source.index(module["path"]) + len(module["path"]) + 1:]
-                    else:
-                        print("Source {} is not inside module path {}".format(source, module["path"]))
-                        sys.exit(1)
-                    if source_file_name in linked_source_file_names:
-                        print("Source {} is a duplicate".format(source))
-                        sys.exit(1)
-
-                    linked_source_file_names.append(source_file_name)
-
-                    # Create any parent directories needed for the source file
-                    symlink_location = os.path.join(linked_directory, source_file_name)
-                    source_dir = os.path.dirname(symlink_location)
-                    if not os.path.exists(source_dir):
-                        os.makedirs(source_dir)
-                    
-                    # Calculate the relative path from the symlink location back to the workspace root
-                    num_parent_dirs = 2 + source_file_name.count(os.path.sep)
-                    relative_prefix = "".join(["../"] * num_parent_dirs)
-                    symlink_target = relative_prefix + source
-                    
-                    os.symlink(symlink_target, symlink_location)
-                    relative_source = source_file_name
-                    if not source.endswith(('.h', '.hpp', '.a')):
-                        combined_lines.append("                \"%s\"," % relative_source)
                 else:
                     print("Source {} is not inside module path {}".format(source, module["path"]))
                     sys.exit(1)
             else:
-                relative_source = source[len(module["path"]) + 1:]
+                # Regular file - must be within module path
-                combined_lines.append("                \"%s\"," % relative_source)
+                if not source.startswith(module["path"]):
+                    print("Source {} is not inside module path {}".format(source, module["path"]))
+                    sys.exit(1)
+                source_file_name = source[len(module["path"]) + 1:]
+
+            # Create symlink for this source file
+            symlink_location = os.path.join(module_directory, source_file_name)
+            source_dir = os.path.dirname(symlink_location)
+            if source_dir and not os.path.exists(source_dir):
+                os.makedirs(source_dir)
+            
+            # Calculate relative path from symlink back to original file
+            # Count directory depth: spm-files/module_name/... -> workspace root
+            num_parent_dirs = symlink_location.count(os.path.sep)
+            relative_prefix = "".join(["../"] * num_parent_dirs)
+            symlink_target = relative_prefix + source
+            
+            # Create the symlink
+            if os.path.lexists(symlink_location):
+                os.unlink(symlink_location)
+            if "arm_arch64_common_macro" in symlink_target:
+                print("Creating symlink from {} to {}".format(symlink_target, symlink_location))
+            os.symlink(symlink_target, symlink_location)
+            
+            # Add to sources list (exclude certain file types)
+            if not source.endswith(('.h', '.hpp', '.a', '.inc')):
+                combined_lines.append("                \"%s\"," % source_file_name)
         combined_lines.append("            ],")
         if module_type == "objc_library" or module_type == "cc_library":
             if len(module["includes"]) == 0:
-                combined_lines.append("            publicHeadersPath: \"\",")
+                # Create dummy headers directory if none specified
+                dummy_headers_path = os.path.join(module_directory, "dummy-headers-path")
+                if not os.path.exists(dummy_headers_path):
+                    os.makedirs(dummy_headers_path)
+                combined_lines.append("            publicHeadersPath: \"dummy-headers-path\",")
             elif len(module["includes"]) == 1:
                 combined_lines.append("            publicHeadersPath: \"%s\"," % module["includes"][0])
             else:
@@ -212,7 +197,7 @@ for name, module in sorted(modules.items()):
                 if cxxopts:
                     combined_lines.append("                .unsafeFlags([")
                     for flag in cxxopts:
-                        if flag.startswith("-std=") and False:
+                        if flag.startswith("-std=") and True:
                             if flag != "-std=c++17":
                                 print("{}: Unsupported C++ standard: {}".format(name, flag))
                                 sys.exit(1)
@@ -267,10 +252,10 @@ for name, module in sorted(modules.items()):
         print("Unknown module type: {}".format(module["type"]))
         sys.exit(1)
 
-combined_lines.append("    ]")
+combined_lines.append("    ],")
-#combined_lines.append("    cxxLanguageStandard: .cxx17")
+combined_lines.append("    cxxLanguageStandard: .cxx17")
 combined_lines.append(")")
 combined_lines.append("")
 
-with open("Package.swift", "w") as f:
+with open("spm-files/Package.swift", "w") as f:
     f.write("\n".join(combined_lines))