Merge pull request 'fix/custom-sysroot' (#125) from fix/custom-sysroot into release/v9.1.0

Reviewed-on: https://git.onlyoffice.com/ONLYOFFICE/build_tools/pulls/125

configure.py

@@ -42,7 +42,7 @@ parser.add_option("--vs-version", action="store", type="string", dest="vs-versio
 parser.add_option("--vs-path", action="store", type="string", dest="vs-path", default="", help="path to vcvarsall")
 parser.add_option("--siteUrl", action="store", type="string", dest="siteUrl", default="127.0.0.1", help="site url")
 parser.add_option("--multiprocess", action="store", type="string", dest="multiprocess", default="1", help="provides ability to specify single process for make")
-parser.add_option("--custom-sysroot", action="store", type="string", dest="custom-sysroot", default="0", help="provides ability to use custom sysroot (ubuntu 16.04) to build c++ core. If value is \"1\", then the sysroot from tools/linux/sysroot will be used, and if it is not there, it will download it and unpack it. You can also set value as the path to the your own sysroot (rarely used). Only for linux")
+parser.add_option("--sysroot", action="store", type="string", dest="sysroot", default="0", help="provides ability to use sysroot (ubuntu 16.04) to build c++ code. If value is \"1\", then the sysroot from tools/linux/sysroot will be used, and if it is not there, it will download it and unpack it. You can also set value as the path to the your own sysroot (rarely used). Only for linux")
 
 (options, args) = parser.parse_args(arguments)
 configOptions = vars(options)

scripts/base.py

@@ -698,9 +698,12 @@ def get_prefix_cross_compiler_arm64():
   return ""
 
 def get_gcc_version():
+  gcc_path = "gcc"
+  if config.option("sysroot") != "":
+    gcc_path = config.option("sysroot") + "/usr/bin/gcc"
   gcc_version_major = 4
   gcc_version_minor = 0
-  gcc_version_str = run_command("gcc -dumpfullversion -dumpversion")['stdout']
+  gcc_version_str = run_command(gcc_path + " -dumpfullversion -dumpversion")['stdout']
   if (gcc_version_str != ""):
     try:
       gcc_ver = gcc_version_str.split(".")
@@ -905,9 +908,23 @@ def _check_icu_common(dir, out):
   return isExist
 
 def qt_copy_icu(out):
-  tests = [get_env("QT_DEPLOY") + "/../lib", "/lib", "/lib/x86_64-linux-gnu", "/lib64", "/lib64/x86_64-linux-gnu"]
+  tests = [get_env("QT_DEPLOY") + "/../lib"]
-  tests += ["/usr/lib", "/usr/lib/x86_64-linux-gnu", "/usr/lib64", "/usr/lib64/x86_64-linux-gnu"]
+  prefix = ""
-  tests += ["/lib/i386-linux-gnu", "/usr/lib/i386-linux-gnu"]
+  postfixes = [""]
+    
+  # TODO add for linux arm desktop build
+  if config.option("sysroot") != "":
+    prefix = config.option("sysroot")
+  else:
+    prefix = ""
+    postfixes += ["/x86_64-linux-gnu"]
+    postfixes += ["/i386-linux-gnu"]
+      
+  for postfix in postfixes:
+    tests += [prefix + "/lib" + postfix]
+    tests += [prefix + "/lib64" + postfix]
+    tests += [prefix + "/usr/lib" + postfix]
+    tests += [prefix + "/usr/lib64" + postfix]
 
   for test in tests:
     if (_check_icu_common(test, out)):

scripts/config.py

@@ -83,6 +83,27 @@ def parse():
 
   if ("windows" == host_platform) and ("2019" == option("vs-version")):
       extend_option("config", "vs2019")
+      
+  # sysroot setup
+  if "linux" != host_platform and "sysroot" in options:
+    options["sysroot"] = ""
+
+  if "linux" == host_platform and "sysroot" in options:
+    if options["sysroot"] == "0":
+      options["sysroot"] = ""
+    elif options["sysroot"] == "1":
+      dst_dir = os.path.abspath(base.get_script_dir(__file__) + '/../tools/linux/sysroot')
+      custom_sysroot = dst_dir + '/sysroot_ubuntu_1604'
+      options["sysroot"] = custom_sysroot
+      if not os.path.isdir(custom_sysroot):
+        print("Sysroot is not found, downloading...")
+        sysroot_url = 'https://github.com/ONLYOFFICE-data/build_tools_data/raw/refs/heads/master/sysroot/sysroot_ubuntu_1604.tar.xz'
+        base.download(sysroot_url, dst_dir + '/sysroot_ubuntu_1604.tar.xz')
+        os.mkdir(custom_sysroot)
+        print("Unpacking...")
+        base.cmd2('tar', ['-xf', dst_dir + '/sysroot_ubuntu_1604.tar.xz', '-C', dst_dir])
+        if os.path.exists(dst_dir + '/sysroot_ubuntu_1604.tar.xz'):
+          os.remove(dst_dir + '/sysroot_ubuntu_1604.tar.xz')
 
   if is_cef_107():
     extend_option("config", "cef_version_107")
@@ -110,29 +131,8 @@ def parse():
   if not "sdkjs-plugin-server" in options:
     options["sdkjs-plugin-server"] = "default"
 
-  # custom-sysroot setup
-  if "linux" != host_platform and "custom-sysroot" in options:
-    options["custom-sysroot"] = ""
-
-  if "linux" == host_platform and "custom-sysroot" in options:
-    if options["custom-sysroot"] == "0":
-      options["custom-sysroot"] = ""
-    elif options["custom-sysroot"] == "1":
-      dst_dir = os.path.abspath(base.get_script_dir(__file__) + '/../tools/linux/sysroot')
-      custom_sysroot = dst_dir + '/sysroot_ubuntu_1604'
-      options["custom-sysroot"] = custom_sysroot
-      if not os.path.isdir(custom_sysroot):
-        print("Custom sysroot is not found, downloading...")
-        sysroot_url = 'https://github.com/ONLYOFFICE-data/build_tools_data/raw/refs/heads/master/sysroot/sysroot_ubuntu_1604.tar.xz'
-        base.download(sysroot_url, dst_dir + '/sysroot_ubuntu_1604.tar.xz')
-        os.mkdir(custom_sysroot)
-        print("Unpacking...")
-        base.cmd2('tar', ['-xf', dst_dir + '/sysroot_ubuntu_1604.tar.xz', '-C', dst_dir])
-        if os.path.exists(dst_dir + '/sysroot_ubuntu_1604.tar.xz'):
-          os.remove(dst_dir + '/sysroot_ubuntu_1604.tar.xz')
-
   if not "arm64-toolchain-bin" in options:
-    if not "custom-sysroot" in options:
+    if not "sysroot" in options:
       options["arm64-toolchain-bin"] = "/usr/bin"
     else:
       options["arm64-toolchain-bin"] = get_custom_sysroot_bin()
@@ -237,12 +237,12 @@ def is_mobile_platform():
   return False
 
 def get_custom_sysroot_bin():
-  return option("custom-sysroot") + "/usr/bin"
+  return option("sysroot") + "/usr/bin"
 
 # todo 32bit support?
 def get_custom_sysroot_lib():
   if base.is_os_64bit():
-    return option("custom-sysroot") + "/usr/lib/x86_64-linux-gnu"
+    return option("sysroot") + "/usr/lib/x86_64-linux-gnu"
 
 def parse_defaults():
   defaults_path = base.get_script_dir() + "/../defaults"

scripts/core_common/modules/boost.py

@@ -99,7 +99,7 @@ def make():
     correct_install_includes_win(base_dir, "win_arm64")
 
   if config.check_option("platform", "linux_64") and not base.is_dir("../build/linux_64"):
-    if config.option("custom-sysroot") == "":
+    if config.option("sysroot") == "":
       addon_config = []
       addon_compile = []
       if "1" == config.option("use-clang"):

scripts/core_common/modules/heif.py

@@ -23,6 +23,28 @@ ANDROID_CMAKE_TOOLCHAIN_FILE = base.get_env("ANDROID_NDK_ROOT") + "/build/cmake/
 # linux arm64 cmake toolchain
 LINUX_ARM64_CMAKE_TOOLCHAIN_FILE = base.get_script_dir() + "/../tools/linux/arm/cross_arm64/linux-arm64.toolchain.cmake"
 
+LINUX_CUSTOM_SYSROOT_TOOLCHAIN_FILE = base.get_script_dir() + "/../tools/linux/sysroot/custom-sysroot.toolchain.cmake"
+
+OLD_ENV = dict()
+
+# get custom sysroot vars as str
+def setup_custom_sysroot_env() -> str:
+  env_vars = []
+  env_vars += ['LD_LIBRARY_PATH=\"' + config.get_custom_sysroot_lib() + "\""]
+  env_vars += ['CC=\"' + config.get_custom_sysroot_bin() + "/gcc\""]
+  env_vars += ['CXX=\"' + config.get_custom_sysroot_bin() + "/g++\""]
+  env_vars += ['AR=\"' + config.get_custom_sysroot_bin() + "/ar\""]
+  env_vars += ['RABLIB=\"' + config.get_custom_sysroot_bin() + "/ranlib\""]
+  env_vars += ['CFLAGS=\"' + "--sysroot=" + config.option("sysroot") + "\""]
+  env_vars += ['CXXFLAGS=\"' + "--sysroot=" + config.option("sysroot") + "\""]
+  env_vars += ['LDFLAGS=\"' + "--sysroot=" + config.option("sysroot") + "\""]
+  
+  env_str = ""
+  for env_var in env_vars:
+    env_str += env_var + " "
+    
+  return env_str
+  
 def get_vs_version():
   vs_version = "14 2015"
   if config.option("vs-version") == "2019":
@@ -69,6 +91,8 @@ def build_with_cmake(platform, cmake_args, build_type):
       cmake_args_ext += ["-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0", "-DCMAKE_OSX_ARCHITECTURES=arm64"]
     elif platform == "linux_arm64":
       cmake_args += ["-DCMAKE_TOOLCHAIN_FILE=" + LINUX_ARM64_CMAKE_TOOLCHAIN_FILE]
+    elif config.option("sysroot") != "":
+      cmake_args += ["-DCMAKE_TOOLCHAIN_FILE=" + LINUX_CUSTOM_SYSROOT_TOOLCHAIN_FILE] # force use custom CXXFLAGS with Release/Debug build
   # IOS
   elif "ios" in platform:
     cmake_args_ext = [
@@ -100,12 +124,16 @@ def build_with_cmake(platform, cmake_args, build_type):
       cmake_args_ext += get_cmake_args_android("x86", "16")
     elif platform == "android_x86_64":
       cmake_args_ext += get_cmake_args_android("x86_64", "21")
+      
+  # env setup for custom sysroot
+  env_str = setup_custom_sysroot_env() if config.option("sysroot") != "" else ""
 
   # run cmake
-  base.cmd("cmake", cmake_args + cmake_args_ext)
+  base.cmd(env_str + "cmake", cmake_args + cmake_args_ext)
+      
   # build
   if "Unix Makefiles" in cmake_args_ext:
-    base.cmd("make", ["-j4"])
+    base.cmd(env_str + "make", ["-j4"])
   else:
     base.cmd("cmake", ["--build", ".", "--config", build_type])
   return

scripts/core_common/modules/icu.py

@@ -133,7 +133,7 @@ def make():
       if "1" == config.option("use-clang"):
         command_configure = "CXXFLAGS=-stdlib=libc++ " + command_configure
         command_compile_addon = "-stdlib=libc++"
-      if "" == config.option("custom-sysroot"):
+      if "" == config.option("sysroot"):
         base.cmd(command_configure, ["Linux", "--prefix=" + base_dir + "/icu/cross_build_install"])
         base.replaceInFile("./../source/icudefs.mk.in", "LDFLAGS = @LDFLAGS@ $(RPATHLDFLAGS)", "LDFLAGS = @LDFLAGS@ $(RPATHLDFLAGS) " + command_compile_addon)
       else:
@@ -143,11 +143,11 @@ def make():
         base.cmd_exe("./../source/configure", ["--prefix=" + base_dir + "/icu/cross_build_install",
                                            "CC=" + config.get_custom_sysroot_bin() + "/gcc", "CXX=" + config.get_custom_sysroot_bin() + "/g++",
                                            "AR=" + config.get_custom_sysroot_bin() + "/ar", "RANLIB=" + config.get_custom_sysroot_bin() + "/ranlib",
-                                           "CFLAGS=--sysroot=" + config.option("custom-sysroot"),
+                                           "CFLAGS=--sysroot=" + config.option("sysroot"),
-                                           "CXXFLAGS=--sysroot=" + config.option("custom-sysroot") + " " + command_compile_addon,
+                                           "CXXFLAGS=--sysroot=" + config.option("sysroot") + " " + command_compile_addon,
-                                           "LDFLAGS=--sysroot=" + config.option("custom-sysroot")])
+                                           "LDFLAGS=--sysroot=" + config.option("sysroot")])
 
-      if "" == config.option("custom-sysroot"):
+      if "" == config.option("sysroot"):
         base.cmd("make", ["-j4"])
         base.cmd("make", ["install"], True)
       else:

scripts/core_common/modules/openssl.py

@@ -97,7 +97,7 @@ def make():
       base.replaceInFile("./Makefile", "CFLAGS=-Wall -O3", "CFLAGS=-Wall -O3 -fvisibility=hidden")
       base.replaceInFile("./Makefile", "CXXFLAGS=-Wall -O3", "CXXFLAGS=-Wall -O3 -fvisibility=hidden -stdlib=libc++")
       base.replaceInFile("./Makefile", "LDFLAGS=", "LDFLAGS=-stdlib=libc++")
-    elif config.option("custom-sysroot") == "":
+    elif config.option("sysroot") == "":
       base.replaceInFile("./Makefile", "CFLAGS=-Wall -O3", "CFLAGS=-Wall -O3 -fvisibility=hidden")
       base.replaceInFile("./Makefile", "CXXFLAGS=-Wall -O3", "CXXFLAGS=-Wall -O3 -fvisibility=hidden")
     else:
@@ -105,10 +105,10 @@ def make():
         ld_library_path_copy = os.environ['LD_LIBRARY_PATH']
       os.environ['LD_LIBRARY_PATH'] = config.get_custom_sysroot_lib()
       base.replaceInFile("./Makefile", "CROSS_COMPILE=", "CROSS_COMPILE=" + config.get_custom_sysroot_bin() + "/")
-      base.replaceInFile("./Makefile", "CFLAGS=-Wall -O3", "CFLAGS=-Wall -O3 -fvisibility=hidden --sysroot=" + config.option("custom-sysroot"),)
+      base.replaceInFile("./Makefile", "CFLAGS=-Wall -O3", "CFLAGS=-Wall -O3 -fvisibility=hidden --sysroot=" + config.option("sysroot"))
-      base.replaceInFile("./Makefile", "CXXFLAGS=-Wall -O3", "CXXFLAGS=-Wall -O3 -fvisibility=hidden --sysroot=" + config.option("custom-sysroot"))
+      base.replaceInFile("./Makefile", "CXXFLAGS=-Wall -O3", "CXXFLAGS=-Wall -O3 -fvisibility=hidden --sysroot=" + config.option("sysroot"))
 
-    if config.option("custom-sysroot") == "":
+    if config.option("sysroot") == "":
       base.cmd("make", [])
       base.cmd("make", ["install"])
       base.cmd("make", ["clean"], True)

scripts/core_common/modules/v8.py

@@ -47,7 +47,7 @@ def is_use_clang():
   gcc_version = base.get_gcc_version()  
     
   is_clang = "false"
-  if config.option("custom-sysroot") == "" and (gcc_version >= 6000 or "1" == config.option("use-clang")):
+  if config.option("sysroot") == "" and (gcc_version >= 6000 or "1" == config.option("use-clang")):
     is_clang = "true"
 
   print("gcc version: " + str(gcc_version) + ", use clang:" + is_clang)

scripts/core_common/modules/v8_89.py

@@ -60,7 +60,7 @@ def make_args(args, platform, is_64=True, is_debug=False):
   
   linux_clang = False
   if platform == "linux":
-    if "" != config.option("custom-sysroot"):
+    if "" != config.option("sysroot"):
       args_copy.append("use_sysroot=false")
       args_copy.append("is_clang=false")
     else:
@@ -207,13 +207,13 @@ def make():
 
   if config.check_option("platform", "linux_64"):
     ld_library_path_copy = ''
-    if config.option("custom-sysroot") != "":
+    if config.option("sysroot") != "":
       if 'LD_LIBRARY_PATH' in os.environ:
         ld_library_path_copy = os.environ['LD_LIBRARY_PATH']
       os.environ['LD_LIBRARY_PATH'] = config.get_custom_sysroot_lib()
 
       src_replace = "config(\"compiler\") {\n  asmflags = []\n  cflags = []\n  cflags_c = []\n  cflags_cc = []\n  cflags_objc = []\n  cflags_objcc = []\n  ldflags = []"
-      dst_replace = "config(\"compiler\") {\n  asmflags = []\n  cflags = [\"--sysroot=" + config.option("custom-sysroot") + "\"]" + "\n  cflags_c = []\n  cflags_cc = [\"--sysroot=" + config.option("custom-sysroot") + "\"]" + "\n  cflags_objc = []\n  cflags_objcc = []\n  ldflags = [\"--sysroot=" + config.option("custom-sysroot") + "\"]"
+      dst_replace = "config(\"compiler\") {\n  asmflags = []\n  cflags = [\"--sysroot=" + config.option("sysroot") + "\"]" + "\n  cflags_c = []\n  cflags_cc = [\"--sysroot=" + config.option("sysroot") + "\"]" + "\n  cflags_objc = []\n  cflags_objcc = []\n  ldflags = [\"--sysroot=" + config.option("sysroot") + "\"]"
       base.replaceInFile("build/config/compiler/BUILD.gn", src_replace, dst_replace)
 
       src_replace = "gcc_toolchain(\"x64\") {\n  cc = \"gcc\"\n  cxx = \"g++\""

scripts/qmake.py

@@ -100,9 +100,10 @@ def make(platform, project, qmake_config_addon="", is_no_errors=False):
     if "1" == config.option("use-clang"):
       build_params.append("-spec")
       build_params.append("linux-clang-libc++")
-    if "" != config.option("custom-sysroot"):
+    if "" != config.option("sysroot"):
       os.environ['LD_LIBRARY_PATH'] = config.get_custom_sysroot_lib()
-      os.environ['QMAKE_CUSTOM_SYSROOT'] = config.option("custom-sysroot")
+      os.environ['QMAKE_CUSTOM_SYSROOT'] = config.option("sysroot")
+      os.environ['PKG_CONFIG_PATH'] = config.get_custom_sysroot_lib() + "/pkgconfig"
       base.cmd_exe(qmake_app, build_params) # calls cmd_exe to pass os.env
     else:
       base.cmd(qmake_app, build_params)

tools/linux/sysroot/build_sysroot.py

@@ -29,7 +29,36 @@ def download_sysroot():
 
     bash_chroot('apt update -y', tmp_sysroot_ubuntu_dir)
     bash_chroot('apt upgrade -y', tmp_sysroot_ubuntu_dir)
-    bash_chroot('apt install -y build-essential curl libpthread-stubs0-dev zlib1g-dev', tmp_sysroot_ubuntu_dir)
+    
+    apt_libs = ["build-essential"]
+    apt_libs += ["libpthread-stubs0-dev"]
+    apt_libs += ["zlib1g-dev"]
+    apt_libs += ["curl"]
+    apt_libs += ["libc6-dev"]
+    apt_libs += ["glib-2.0-dev"]
+    apt_libs += ["libglu1-mesa-dev"]
+    apt_libs += ["libgtk-3-dev"]
+    apt_libs += ["libpulse-dev"]
+    apt_libs += ["libasound2-dev"]
+    apt_libs += ["libatspi2.0-dev"]
+    apt_libs += ["libcups2-dev"]
+    apt_libs += ["libdbus-1-dev"]
+    apt_libs += ["libicu-dev"]
+    apt_libs += ["libgstreamer1.0-dev"]
+    apt_libs += ["libgstreamer-plugins-base1.0-dev"]
+    apt_libs += ["libx11-xcb-dev"]
+    apt_libs += ["libxcb*"]
+    apt_libs += ["libxi-dev"]
+    apt_libs += ["libxrender-dev"]
+    apt_libs += ["libxss-dev"]
+    apt_libs += ["libxkbcommon-dev"]
+    apt_libs += ["libxkbcommon-x11-dev"]
+    apt_libs += ["libnotify-dev"]
+    
+    apt_libs_str = ""
+    for apt_lib in apt_libs:
+        apt_libs_str += apt_lib + " "
+    bash_chroot('apt install -y ' + apt_libs_str, tmp_sysroot_ubuntu_dir)
 
     # # downloading arm toolchain
     arm_toolchain_url = 'https://releases.linaro.org/components/toolchain/binaries/5.4-2017.05/aarch64-linux-gnu/'

tools/linux/sysroot/custom-sysroot.toolchain.cmake

@@ -0,0 +1,4 @@
+set(CMAKE_C_FLAGS $ENV{CFLAGS})
+set(CMAKE_CXX_FLAGS $ENV{CXXFLAGS})
+set(CMAKE_C_LINKER_FLAGS $ENV{LDFLAGS})
+set(CMAKE_CXX_LINKER_FLAGS $ENV{LDFLAGS})

tools/linux/sysroot/fix_symlinks.py

@@ -4,46 +4,42 @@ import uuid
 
 # change symbolic link to relative paths
 def fix_symlinks(top_dir='./sysroot_ubuntu_1604'):
-    top_dir += '/usr/lib/x86_64-linux-gnu'
     for root, dirs, files in os.walk(top_dir):
         for name in files:
             path = os.path.join(root, name)
             if not os.path.islink(path):
                 continue
-
             try:
                 target = os.readlink(path)
             except OSError as e:
                 print(f"Error reading link '{path}': {e}", file=sys.stderr)
                 continue
-
+            
-            if not target.startswith('/lib/'):
+            if not target.startswith('/'):
                 continue
-
+            
-            new_target = "../../../lib" + target[4:]
+            new_target = top_dir + target
+            new_target_rel = os.path.relpath(new_target, os.path.dirname(path))
             temp_name = f".tmp.{uuid.uuid4().hex}"
             temp_path = os.path.join(root, temp_name)
-
             try:
-                os.symlink(new_target, temp_path)
+                os.symlink(new_target_rel, temp_path)
             except OSError as e:
                 print(f"Failed to create temporary symlink for '{path}': {e}", file=sys.stderr)
                 continue
-
             try:
                 os.replace(temp_path, path)
-                print(f"Updated: {path} -> {new_target}")
+                print(f"Updated: {path} -> {new_target_rel}")
             except OSError as e:
                 print(f"Failed to replace symlink '{path}': {e}", file=sys.stderr)
                 try:
                     os.unlink(temp_path)
                 except OSError:
                     pass
-        break # no subfolders
 
 if __name__ == "__main__":
     if len(sys.argv) > 1:
         directory = sys.argv[1]
     else:
         directory = './sysroot_ubuntu_1604'
-    fix_symlinks(directory)
+    fix_symlinks(directory)