From 7770a41f084ffd497531d9d68e6f21e8bd2750fd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 2 Aug 2023 12:14:58 +0000 Subject: [PATCH 1/5] Update hard-coded version to v7.5.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 815da58..18bb418 100644 --- a/version +++ b/version @@ -1 +1 @@ -7.4.1 +7.5.0 From 8ea37f2b0353381cbd85f9f543a9e9047069c610 Mon Sep 17 00:00:00 2001 From: Semyon Bezrukov Date: Thu, 3 Aug 2023 11:00:15 +0300 Subject: [PATCH 2/5] MacOS package build refactoring (#718) --- scripts/package_desktop.py | 75 +++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 41 deletions(-) diff --git a/scripts/package_desktop.py b/scripts/package_desktop.py index 74dbb3f..209d283 100644 --- a/scripts/package_desktop.py +++ b/scripts/package_desktop.py @@ -366,7 +366,7 @@ def make_advinst(): def make_macos(): global package_name, build_dir, branding_dir, updates_dir, changes_dir, \ - suffix, lane, scheme, app_version + suffix, lane, scheme, released_updates_dir package_name = branding.desktop_package_name build_dir = branding.desktop_build_dir branding_dir = branding.desktop_branding_dir @@ -379,6 +379,7 @@ def make_macos(): }[common.platform] lane = "release_" + suffix scheme = package_name + "-" + suffix + sparkle_updates = False utils.set_cwd(branding_dir) @@ -394,32 +395,38 @@ def make_macos(): + "/desktopeditors/editors/web-apps/apps/*/main/resources/help"): utils.delete_dir(path) - appcast_url = branding.sparkle_base_url + "/" + suffix + "/" + branding.desktop_package_name.lower() + ".xml" - release_bundle_version_string = utils.sh_output( - 'curl -Ls ' + appcast_url + ' 2> /dev/null' \ - + ' | xmllint --xpath "/rss/channel/item[1]/*[name()=\'sparkle:shortVersionString\']/text()" -', - verbose=True).rstrip() - release_bundle_version = utils.sh_output( - 'curl -Ls ' + appcast_url + ' 2> /dev/null' \ - + ' | xmllint --xpath "/rss/channel/item[1]/*[name()=\'sparkle:version\']/text()" -', - verbose=True).rstrip() + if utils.get_env("ARCHIVES_DIR"): + sparkle_updates = True + released_updates_dir = "%s/%s/_updates" % (utils.get_env("ARCHIVES_DIR"), scheme) + plistbuddy = "/usr/libexec/PlistBuddy" + plist_path = "%s/%s/ONLYOFFICE/Resources/%s-%s/Info.plist" \ + % (common.workspace_dir, branding_dir, package_name, suffix) - app_version = common.version - bundle_version = str(int(release_bundle_version) + 1) - plist_path = "%s/%s/ONLYOFFICE/Resources/%s-%s/Info.plist" \ - % (common.workspace_dir, branding.desktop_branding_dir, branding.desktop_package_name, suffix) - utils.sh('/usr/libexec/PlistBuddy -c "Set :CFBundleShortVersionString %s" %s' \ - % (common.version, plist_path), verbose=True) - utils.sh('/usr/libexec/PlistBuddy -c "Set :CFBundleVersion %s" %s' \ - % (bundle_version, plist_path), verbose=True) - utils.sh('/usr/libexec/PlistBuddy -c "Add :ASCWebappsHelpUrl string %s" %s' \ - % ("https://download.onlyoffice.com/install/desktop/editors/help/v" + app_version + "/apps", plist_path), verbose=True) + appcast = utils.sh_output('%s -c "Print :SUFeedURL" %s' \ + % (plistbuddy, plist_path), verbose=True).rstrip() + appcast = released_updates_dir + "/" + appcast[appcast.rfind("/")+1:] - utils.log("RELEASE=" + release_bundle_version_string + "(" + release_bundle_version + ")" \ - + "\nCURRENT=" + common.version + "(" + bundle_version + ")") + release_version_string = utils.sh_output( + 'xmllint --xpath "/rss/channel/item[1]/*[name()=\'sparkle:shortVersionString\']/text()" ' + appcast, + verbose=True).rstrip() + release_version = utils.sh_output( + 'xmllint --xpath "/rss/channel/item[1]/*[name()=\'sparkle:version\']/text()" ' + appcast, + verbose=True).rstrip() + bundle_version = str(int(release_version) + 1) + help_url = "https://download.onlyoffice.com/install/desktop/editors/help/v" + common.version + "/apps" + + utils.sh('%s -c "Set :CFBundleShortVersionString %s" %s' \ + % (plistbuddy, common.version, plist_path), verbose=True) + utils.sh('%s -c "Set :CFBundleVersion %s" %s' \ + % (plistbuddy, bundle_version, plist_path), verbose=True) + utils.sh('%s -c "Add :ASCWebappsHelpUrl string %s" %s' \ + % (plistbuddy, help_url, plist_path), verbose=True) + + utils.log("RELEASE=" + release_version_string + "(" + release_version + ")" \ + + "\nCURRENT=" + common.version + "(" + bundle_version + ")") dmg = make_dmg() - if dmg: + if dmg and sparkle_updates: make_sparkle_updates() utils.set_cwd(common.workspace_dir) @@ -446,7 +453,7 @@ def make_dmg(): utils.log_h2("desktop zip deploy") ret = aws_s3_upload( - ["build/%s-%s.zip" % (scheme, app_version)], + ["build/%s-%s.zip" % (scheme, common.version)], "mac/%s/%s/%s/" % (common.version, common.build, suffix), "Archive" ) @@ -456,14 +463,13 @@ def make_dmg(): def make_sparkle_updates(): utils.log_h2("desktop sparkle files build") - zip_filename = scheme + '-' + app_version + zip_filename = scheme + '-' + common.version macos_zip = "build/" + zip_filename + ".zip" - updates_storage_dir = "%s/%s/_updates" % (utils.get_env('ARCHIVES_DIR'), scheme) utils.create_dir(updates_dir) utils.copy_file(macos_zip, updates_dir) - utils.copy_dir_content(updates_storage_dir, updates_dir, ".zip") + utils.copy_dir_content(released_updates_dir, updates_dir, ".zip") - for file in utils.glob_path(changes_dir + "/" + app_version + "/*.html"): + for file in utils.glob_path(changes_dir + "/" + common.version + "/*.html"): filename = utils.get_basename(file).replace("changes", zip_filename) utils.copy_file(file, updates_dir + "/" + filename) @@ -478,19 +484,6 @@ def make_sparkle_updates(): ) utils.set_summary("desktop sparkle files build", ret) - # utils.log_h3("edit sparkle appcast links") - # appcast_url = branding.sparkle_base_url + "/" + suffix - # appcast = "%s/%s.xml" % (updates_dir, package_name.lower()) - # for lang, base in update_changes_list.items(): - # if base == "ReleaseNotes": - # utils.replace_in_file(appcast, - # r'(.+/).+(\.html)', - # "\\1" + base + "\\2") - # else: - # utils.replace_in_file(appcast, - # r'().+(\.html)', - # "\\1" + base + "\\2") - utils.log("") utils.log_h3("generate checksums") utils.sh( From 88843a1f2d855ab3690aca75fcd4eabdca312621 Mon Sep 17 00:00:00 2001 From: Semyon Bezrukov Date: Thu, 3 Aug 2023 17:40:20 +0300 Subject: [PATCH 3/5] Fix windows package build (#719) --- scripts/package_desktop.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/scripts/package_desktop.py b/scripts/package_desktop.py index 209d283..65b3ce4 100644 --- a/scripts/package_desktop.py +++ b/scripts/package_desktop.py @@ -54,7 +54,7 @@ def aws_s3_upload(files, key, ptype=None): def make_windows(): global package_version, arch_list, source_dir, desktop_dir, viewer_dir, \ - inno_file, inno_help_file, inno_sa_file, inno_update_file, advinst_file, zip_file + inno_file, inno_sa_file, inno_update_file, advinst_file utils.set_cwd("desktop-apps\\win-linux\\package\\windows") package_name = branding.desktop_package_name @@ -67,9 +67,7 @@ def make_windows(): } suffix = arch_list[common.platform] if common.platform.endswith("_xp"): suffix += "-xp" - zip_file = "%s-%s-%s.zip" % (package_name, package_version, suffix) inno_file = "%s-%s-%s.exe" % (package_name, package_version, suffix) - inno_help_file = "%s-Help-%s-%s.exe" % (package_name, package_version, suffix) inno_sa_file = "%s-Standalone-%s-%s.exe" % (package_name, package_version, suffix) inno_update_file = "update\\editors_update_%s.exe" % suffix.replace("-","_") advinst_file = "%s-%s-%s.msi" % (package_name, package_version, suffix) @@ -127,7 +125,7 @@ def make_zip(): utils.log_h2("desktop zip build") args = [ - "-OutFile", zip_file, + "-Target", common.platform, "-BuildDir", "build", "-DesktopDir", branding.desktop_product_name_s ] @@ -137,14 +135,12 @@ def make_zip(): args += ["-ExcludeHelp"] if common.sign: args += ["-Sign", "-CertName", branding.cert_name] - ret = utils.ps1( - "make_zip.ps1", args, creates=zip_file, verbose=True - ) + ret = utils.ps1("make_zip.ps1", args, verbose=True) utils.set_summary("desktop zip build", ret) if common.deploy and ret: utils.log_h2("desktop zip deploy") - ret = aws_s3_upload([zip_file], "win/generic/", "Portable") + ret = aws_s3_upload(utils.glob_path("*.zip"), "win/generic/", "Portable") utils.set_summary("desktop zip deploy", ret) return @@ -195,10 +191,6 @@ def make_inno(): utils.set_summary("desktop inno build", ret) if branding.onlyoffice and not common.platform.endswith("_xp"): - args = ["iscc"] + iscc_args + ["help.iss"] - ret = utils.cmd(*args, creates=inno_help_file, verbose=True) - utils.set_summary("desktop inno help build", ret) - args = ["iscc"] + iscc_args + ["/DEMBED_HELP", "/DsPackageEdition=Standalone", "common.iss"] ret = utils.cmd(*args, creates=inno_sa_file, verbose=True) utils.set_summary("desktop inno standalone build", ret) @@ -214,10 +206,6 @@ def make_inno(): utils.set_summary("desktop inno deploy", ret) if branding.onlyoffice and not common.platform.endswith("_xp"): - utils.log_h2("desktop inno help deploy") - ret = aws_s3_upload([inno_help_file], "win/inno/","Installer") - utils.set_summary("desktop inno help deploy", ret) - utils.log_h2("desktop inno standalone deploy") ret = aws_s3_upload([inno_sa_file], "win/inno/","Installer") utils.set_summary("desktop inno standalone deploy", ret) From b9ccd9849a7d3f65b1af4a43b854b18e823431e1 Mon Sep 17 00:00:00 2001 From: Semyon Bezrukov Date: Fri, 4 Aug 2023 17:59:21 +0300 Subject: [PATCH 4/5] Refactoring packages deploy (#720) * Refactoring packages deploy * Small fix --- make_package.py | 29 +++---- scripts/package_builder.py | 66 ++++++--------- scripts/package_common.py | 14 ---- scripts/package_core.py | 66 ++++++--------- scripts/package_desktop.py | 161 +++++++++++++------------------------ scripts/package_mobile.py | 27 +++---- scripts/package_server.py | 72 ++++++----------- scripts/package_utils.py | 93 ++++++++++++++++----- 8 files changed, 229 insertions(+), 299 deletions(-) diff --git a/make_package.py b/make_package.py index 713e116..42e16b5 100755 --- a/make_package.py +++ b/make_package.py @@ -22,7 +22,6 @@ parser.add_argument("-B", "--build", dest="build", type=str, args = parser.parse_args() # vars -common.workspace_dir = utils.get_abspath(utils.get_script_dir(__file__) + "/..") common.os_family = utils.host_platform() common.platform = args.platform common.prefix = common.platformPrefixes[common.platform] if common.platform in common.platformPrefixes else "" @@ -30,14 +29,14 @@ common.targets = args.targets common.clean = "clean" in args.targets common.sign = "sign" in args.targets common.deploy = "deploy" in args.targets -common.version = args.version if (args.version is not None) else utils.get_env("PRODUCT_VERSION", "1.0.0") -common.build = args.build if (args.build is not None) else utils.get_env("BUILD_NUMBER", "1") -common.channel = utils.get_env("BUILD_CHANNEL", "other") +common.version = args.version if args.version else utils.get_env("BUILD_VERSION", "0.0.0") +common.build = args.build if args.build else utils.get_env("BUILD_NUMBER", "0") common.branding = args.branding common.timestamp = utils.get_timestamp() +common.workspace_dir = utils.get_abspath(utils.get_script_dir(__file__) + "/..") +common.branding_dir = utils.get_abspath(common.workspace_dir + "/" + args.branding) if args.branding else common.workspace_dir +common.deploy_data = utils.get_path(common.workspace_dir + "/deploy.txt") common.summary = [] -common.deploy_data = [] -utils.log("workspace_dir: " + common.workspace_dir) utils.log("os_family: " + common.os_family) utils.log("platform: " + str(common.platform)) utils.log("prefix: " + str(common.prefix)) @@ -49,6 +48,8 @@ utils.log("version: " + common.version) utils.log("build: " + common.build) utils.log("branding: " + str(common.branding)) utils.log("timestamp: " + common.timestamp) +utils.log("workspace_dir: " + common.workspace_dir) +utils.log("branding_dir: " + common.branding_dir) # branding if common.branding is not None: @@ -63,22 +64,22 @@ import package_mobile # build utils.set_cwd(common.workspace_dir, verbose=True) -utils.delete_file("deploy.json") +utils.delete_file(common.deploy_data) if "core" in common.targets: package_core.make() -if "closure-maps-os" in common.targets: - package_core.deploy_closure_maps("opensource") -if "closure-maps-com" in common.targets: - package_core.deploy_closure_maps("commercial") +if "closuremaps_opensource" in common.targets: + package_core.deploy_closuremaps("opensource") +if "closuremaps_commercial" in common.targets: + package_core.deploy_closuremaps("commercial") if "desktop" in common.targets: package_desktop.make() if "builder" in common.targets: package_builder.make() -if "server-community" in common.targets: +if "server_community" in common.targets: package_server.make("community") -if "server-enterprise" in common.targets: +if "server_enterprise" in common.targets: package_server.make("enterprise") -if "server-developer" in common.targets: +if "server_developer" in common.targets: package_server.make("developer") if "mobile" in common.targets: package_mobile.make() diff --git a/scripts/package_builder.py b/scripts/package_builder.py index 858016c..c6ae382 100644 --- a/scripts/package_builder.py +++ b/scripts/package_builder.py @@ -17,33 +17,20 @@ def make(): utils.log("Unsupported host OS") return -def aws_s3_upload(files, key, ptype=None): - if not files: - return False +def s3_upload(files, dst): + if not files: return False ret = True - key = "builder/" + key - for file in files: - if not utils.is_file(file): - utils.log_err("file not exist: " + file) - ret &= False - continue - args = ["aws"] + for f in files: + key = dst + utils.get_basename(f) if dst.endswith("/") else dst + aws_kwargs = { "acl": "public-read" } if hasattr(branding, "s3_endpoint_url"): - args += ["--endpoint-url=" + branding.s3_endpoint_url] - args += [ - "s3", "cp", "--no-progress", "--acl", "public-read", - "--metadata", "md5=" + utils.get_md5(file), - file, "s3://" + branding.s3_bucket + "/" + key - ] - if common.os_family == "windows": - upload = utils.cmd(*args, verbose=True) - else: - upload = utils.sh(" ".join(args), verbose=True) + aws_kwargs["endpoint_url"] = branding.s3_endpoint_url + upload = utils.s3_upload( + f, "s3://" + branding.s3_bucket + "/" + key, **aws_kwargs) + if upload: + utils.add_deploy_data(key) + utils.log("URL: " + branding.s3_base_url + "/" + key) ret &= upload - if upload and ptype is not None: - full_key = key - if full_key.endswith("/"): full_key += utils.get_basename(file) - utils.add_deploy_data("builder", ptype, file, full_key) return ret def make_windows(): @@ -87,7 +74,7 @@ def make_zip(): if common.deploy and ret: utils.log_h2("builder zip deploy") - ret = aws_s3_upload(["build\\" + zip_file], "win/generic/", "Portable") + ret = s3_upload(["build\\" + zip_file], "builder/win/generic/") utils.set_summary("builder zip deploy", ret) return @@ -116,7 +103,7 @@ def make_inno(): if common.deploy and ret: utils.log_h2("builder inno deploy") - ret = aws_s3_upload(["build\\" + inno_file], "win/inno/", "Installer") + ret = s3_upload(["build\\" + inno_file], "builder/win/inno/") utils.set_summary("builder inno deploy", ret) return @@ -144,7 +131,7 @@ def make_macos(): if common.deploy and ret: utils.log_h2("builder deploy") - ret = aws_s3_upload([builder_tar], "mac/", "Portable") + ret = s3_upload([builder_tar], "builder/mac/") utils.set_summary("builder deploy", ret) utils.set_cwd(common.workspace_dir) @@ -162,32 +149,25 @@ def make_linux(): ret = utils.sh("make clean && make " + " ".join(make_args), verbose=True) utils.set_summary("builder build", ret) - rpm_arch = "x86_64" - if common.platform == "linux_aarch64": rpm_arch = "aarch64" - if common.deploy: - utils.log_h2("builder deploy") if ret: if "tar" in branding.builder_make_targets: utils.log_h2("builder tar deploy") - ret = aws_s3_upload( - utils.glob_path("tar/*.tar.gz"), - "linux/generic/", "Portable" - ) + ret = s3_upload( + utils.glob_path("tar/*.tar.gz"), + "builder/linux/generic/") utils.set_summary("builder tar deploy", ret) if "deb" in branding.builder_make_targets: utils.log_h2("builder deb deploy") - ret = aws_s3_upload( - utils.glob_path("deb/*.deb"), - "linux/debian/", "Debian" - ) + ret = s3_upload( + utils.glob_path("deb/*.deb"), + "builder/linux/debian/") utils.set_summary("builder deb deploy", ret) if "rpm" in branding.builder_make_targets: utils.log_h2("builder rpm deploy") - ret = aws_s3_upload( - utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), - "linux/rhel/", "CentOS" - ) + ret = s3_upload( + utils.glob_path("rpm/builddir/RPMS/*/*.rpm"), + "builder/linux/rhel/") utils.set_summary("builder rpm deploy", ret) else: if "tar" in branding.builder_make_targets: diff --git a/scripts/package_common.py b/scripts/package_common.py index e71e223..08732e8 100644 --- a/scripts/package_common.py +++ b/scripts/package_common.py @@ -1,19 +1,5 @@ #!/usr/bin/env python -platformTitles = { - "windows_x64": "Windows x64", - "windows_x86": "Windows x86", - "windows_x64_xp": "Windows x64 XP", - "windows_x86_xp": "Windows x86 XP", - "darwin_x86_64": "macOS x86_64", - "darwin_arm64": "macOS arm64", - "darwin_x86_64_v8": "macOS x86_64 V8", - "linux_x86_64": "Linux x86_64", - "linux_aarch64": "Linux aarch64", - "linux_x86_64_cef": "Linux x86_64 cef107", - "android": "Android", -} - platformPrefixes = { "windows_x64": "win_64", "windows_x86": "win_32", diff --git a/scripts/package_core.py b/scripts/package_core.py index 4f68c8c..cc17dd7 100644 --- a/scripts/package_core.py +++ b/scripts/package_core.py @@ -26,8 +26,8 @@ def make_core(): repo = repos[common.platform] branch = utils.get_env("BRANCH_NAME") core_7z = utils.get_path("build_tools/out/%s/%s/core.7z" % (prefix, company)) - dest_version = "%s/core/%s/%s/%s/" % (repo["repo"], branch, repo["version"], repo["arch"]) - dest_latest = "%s/core/%s/%s/%s/" % (repo["repo"], branch, "latest", repo["arch"]) + dest_version = "%s/core/%s/%s/%s" % (repo["repo"], branch, repo["version"], repo["arch"]) + dest_latest = "%s/core/%s/%s/%s" % (repo["repo"], branch, "latest", repo["arch"]) if branch is None: utils.log_err("BRANCH_NAME variable is undefined") @@ -39,62 +39,48 @@ def make_core(): return utils.log_h2("core deploy") - args = ["aws", "s3", "cp", "--acl", "public-read", "--no-progress", - "--metadata", "md5=" + utils.get_md5(core_7z), - core_7z, "s3://" + branding.s3_bucket + "/" + dest_version + "core.7z"] - if common.os_family == "windows": - ret = utils.cmd(*args, verbose=True) - else: - ret = utils.sh(" ".join(args), verbose=True) + aws_kwargs = { "acl": "public-read" } + if hasattr(branding, "s3_endpoint_url"): + aws_kwargs["endpoint_url"]=branding.s3_endpoint_url + ret = utils.s3_upload( + core_7z, + "s3://" + branding.s3_bucket + "/" + dest_version + "/core.7z", + **aws_kwargs) if ret: - utils.add_deploy_data("core", "Archive", core_7z, dest_version + "core.7z") - args = ["aws", "s3", "sync", "--delete", - "--acl", "public-read", "--no-progress", - "s3://" + branding.s3_bucket + "/" + dest_version, - "s3://" + branding.s3_bucket + "/" + dest_latest] - if common.os_family == "windows": - ret &= utils.cmd(*args, verbose=True) - else: - ret &= utils.sh(" ".join(args), verbose=True) + utils.log("URL: " + branding.s3_base_url + "/" + dest_version + "/core.7z") + utils.add_deploy_data(dest_version + "/core.7z") + ret = utils.s3_sync( + "s3://" + branding.s3_bucket + "/" + dest_version + "/", + "s3://" + branding.s3_bucket + "/" + dest_latest + "/", + delete=True, **aws_kwargs) + utils.log("URL: " + branding.s3_base_url + "/" + dest_latest + "/core.7z") utils.set_summary("core deploy", ret) return -def deploy_closure_maps(license): +def deploy_closuremaps(license): if not common.deploy: return utils.log_h1("CLOSURE MAPS") utils.set_cwd(utils.get_path("sdkjs/build/maps")) - branch = utils.get_env("BRANCH_NAME") maps = utils.glob_path("*.js.map") - - if branch is None: - utils.log_err("BRANCH_NAME variable is undefined") - utils.set_summary("closure maps " + license + " deploy", False) - return - if not maps: utils.log_err("files do not exist") utils.set_summary("closure maps " + license + " deploy", False) return utils.log_h2("closure maps " + license + " deploy") - dest = "closure-maps/%s/%s/%s" % (common.version, common.build, license) + aws_kwargs = {} + if hasattr(branding, "s3_endpoint_url"): + aws_kwargs["endpoint_url"]=branding.s3_endpoint_url ret = True - for file in maps: - args = ["aws"] - if hasattr(branding, "s3_endpoint_url"): - args += ["--endpoint-url=" + branding.s3_endpoint_url] - args += [ - "s3", "cp", "--no-progress", "--metadata", "md5=" + utils.get_md5(file), - file, "s3://" + branding.s3_bucket + "/" + dest + "/" - ] - if common.os_family == "windows": - upload = utils.cmd(*args, verbose=True) - else: - upload = utils.sh(" ".join(args), verbose=True) + for f in maps: + key = "closuremaps/%s/%s/%s/%s" % (license, common.version, common.build, f) + upload = utils.s3_upload( + f, "s3://" + branding.s3_bucket + "/" + key, **aws_kwargs) ret &= upload if upload: - utils.add_deploy_data("core", "Closure maps " + license, file, dest + "/" + file) + utils.log("URL: " + branding.s3_base_url + "/" + key) + utils.add_deploy_data(key) utils.set_summary("closure maps " + license + " deploy", ret) utils.set_cwd(common.workspace_dir) diff --git a/scripts/package_desktop.py b/scripts/package_desktop.py index 65b3ce4..4bcb3f8 100644 --- a/scripts/package_desktop.py +++ b/scripts/package_desktop.py @@ -19,33 +19,20 @@ def make(): utils.log("Unsupported host OS") return -def aws_s3_upload(files, key, ptype=None): - if not files: - return False +def s3_upload(files, dst): + if not files: return False ret = True - key = "desktop/" + key - for file in files: - if not utils.is_file(file): - utils.log_err("file not exist: " + file) - ret &= False - continue - args = ["aws"] + for f in files: + key = dst + utils.get_basename(f) if dst.endswith("/") else dst + aws_kwargs = { "acl": "public-read" } if hasattr(branding, "s3_endpoint_url"): - args += ["--endpoint-url=" + branding.s3_endpoint_url] - args += [ - "s3", "cp", "--no-progress", "--acl", "public-read", - "--metadata", "md5=" + utils.get_md5(file), - file, "s3://" + branding.s3_bucket + "/" + key - ] - if common.os_family == "windows": - upload = utils.cmd(*args, verbose=True) - else: - upload = utils.sh(" ".join(args), verbose=True) + aws_kwargs["endpoint_url"] = branding.s3_endpoint_url + upload = utils.s3_upload( + f, "s3://" + branding.s3_bucket + "/" + key, **aws_kwargs) + if upload: + utils.add_deploy_data(key) + utils.log("URL: " + branding.s3_base_url + "/" + key) ret &= upload - if upload and ptype is not None: - full_key = key - if full_key.endswith("/"): full_key += utils.get_basename(file) - utils.add_deploy_data("desktop", ptype, file, full_key) return ret # @@ -54,7 +41,7 @@ def aws_s3_upload(files, key, ptype=None): def make_windows(): global package_version, arch_list, source_dir, desktop_dir, viewer_dir, \ - inno_file, inno_sa_file, inno_update_file, advinst_file + inno_file, inno_sa_file, inno_update_file, inno_update_file_new, advinst_file utils.set_cwd("desktop-apps\\win-linux\\package\\windows") package_name = branding.desktop_package_name @@ -70,6 +57,7 @@ def make_windows(): inno_file = "%s-%s-%s.exe" % (package_name, package_version, suffix) inno_sa_file = "%s-Standalone-%s-%s.exe" % (package_name, package_version, suffix) inno_update_file = "update\\editors_update_%s.exe" % suffix.replace("-","_") + inno_update_file_new = "%s-Update-%s-%s.exe" % (package_name, package_version, suffix) advinst_file = "%s-%s-%s.msi" % (package_name, package_version, suffix) if common.clean: @@ -140,7 +128,7 @@ def make_zip(): if common.deploy and ret: utils.log_h2("desktop zip deploy") - ret = aws_s3_upload(utils.glob_path("*.zip"), "win/generic/", "Portable") + ret = s3_upload(utils.glob_path("*.zip"), "desktop/win/generic/") utils.set_summary("desktop zip deploy", ret) return @@ -202,27 +190,21 @@ def make_inno(): if common.deploy: utils.log_h2("desktop inno deploy") - ret = aws_s3_upload([inno_file], "win/inno/","Installer") + ret = s3_upload([inno_file], "desktop/win/inno/") utils.set_summary("desktop inno deploy", ret) if branding.onlyoffice and not common.platform.endswith("_xp"): utils.log_h2("desktop inno standalone deploy") - ret = aws_s3_upload([inno_sa_file], "win/inno/","Installer") + ret = s3_upload([inno_sa_file], "desktop/win/inno/") utils.set_summary("desktop inno standalone deploy", ret) utils.log_h2("desktop inno update deploy") if utils.is_file(inno_update_file): - ret = aws_s3_upload( - [inno_update_file], - "win/inno/%s/%s/" % (common.version, common.build), - "Update" - ) + ret = s3_upload( + [inno_update_file], "desktop/win/inno/" + inno_update_file_new) elif utils.is_file(inno_file): - ret = aws_s3_upload( - [inno_file], - "win/inno/%s/%s/%s" % (common.version, common.build, utils.get_basename(inno_update_file)), - "Update" - ) + ret = s3_upload( + [inno_file], "desktop/win/inno/" + inno_update_file_new) else: ret = False utils.set_summary("desktop inno update deploy", ret) @@ -235,11 +217,9 @@ def make_update_files(): if common.deploy and utils.glob_path(changes_dir + "\\*.html"): utils.log_h2("desktop update files deploy") - ret = aws_s3_upload( - utils.glob_path(changes_dir + "\\*.html"), - "win/update/%s/%s/" % (common.version, common.build), - "Update" - ) + ret = s3_upload( + utils.glob_path(changes_dir + "\\*.html"), + "desktop/win/update/%s/%s/" % (common.version, common.build)) utils.set_summary("desktop update files deploy", ret) return @@ -344,7 +324,7 @@ def make_advinst(): if common.deploy and ret: utils.log_h2("desktop advinst deploy") - ret = aws_s3_upload([advinst_file], "win/advinst/", "Installer") + ret = s3_upload([advinst_file], "desktop/win/advinst/") utils.set_summary("desktop advinst deploy", ret) return @@ -376,6 +356,7 @@ def make_macos(): utils.delete_dir(utils.get_env("HOME") + "/Library/Developer/Xcode/Archives") utils.delete_dir(utils.get_env("HOME") + "/Library/Caches/Sparkle_generate_appcast") + utils.log_h2("build") source_dir = "%s/build_tools/out/%s/%s" \ % (common.workspace_dir, common.prefix, branding.company_name) if branding.onlyoffice: @@ -432,19 +413,15 @@ def make_dmg(): if common.deploy and dmg: utils.log_h2("desktop dmg deploy") - ret = aws_s3_upload( - utils.glob_path("build/*.dmg"), - "mac/%s/%s/%s/" % (common.version, common.build, suffix), - "Disk Image" - ) + ret = s3_upload( + utils.glob_path("build/*.dmg"), + "desktop/mac/%s/%s/%s/" % (suffix, common.version, common.build)) utils.set_summary("desktop dmg deploy", ret) utils.log_h2("desktop zip deploy") - ret = aws_s3_upload( - ["build/%s-%s.zip" % (scheme, common.version)], - "mac/%s/%s/%s/" % (common.version, common.build, suffix), - "Archive" - ) + ret = s3_upload( + ["build/%s-%s.zip" % (scheme, common.version)], + "desktop/mac/%s/%s/%s/" % (suffix, common.version, common.build)) utils.set_summary("desktop zip deploy", ret) return dmg @@ -485,22 +462,12 @@ def make_sparkle_updates(): if common.deploy: utils.log_h2("desktop sparkle files deploy") - ret = aws_s3_upload( - utils.glob_path("build/update/*.delta") \ - + utils.glob_path("build/update/*.xml") \ - + utils.glob_path("build/update/*.html"), - "mac/%s/%s/%s/" % (common.version, common.build, suffix), - "Sparkle" - ) + ret = s3_upload( + utils.glob_path("build/update/*.delta") \ + + utils.glob_path("build/update/*.xml") \ + + utils.glob_path("build/update/*.html"), + "desktop/mac/%s/%s/%s/" % (suffix, common.version, common.build)) utils.set_summary("desktop sparkle files deploy", ret) - - utils.log_h2("desktop checksums deploy") - ret = aws_s3_upload( - utils.glob_path("build/update/*.txt"), - "mac/%s/%s/%s/" % (common.version, common.build, suffix), - "Checksums" - ) - utils.set_summary("desktop checksums deploy", ret) return # @@ -519,64 +486,52 @@ def make_linux(): ret = utils.sh("make clean && make " + " ".join(make_args), verbose=True) utils.set_summary("desktop build", ret) - rpm_arch = "x86_64" + rpm_arch = "*" if common.platform == "linux_aarch64": rpm_arch = "aarch64" if common.deploy: - utils.log_h2("desktop deploy") if ret: utils.log_h2("desktop tar deploy") if "tar" in branding.desktop_make_targets: - ret = aws_s3_upload( - utils.glob_path("tar/*.tar*"), - "linux/generic/", "Portable" - ) + ret = s3_upload( + utils.glob_path("tar/*.tar*"), + "desktop/linux/generic/") utils.set_summary("desktop tar deploy", ret) if "deb" in branding.desktop_make_targets: utils.log_h2("desktop deb deploy") - ret = aws_s3_upload( - utils.glob_path("deb/*.deb"), - "linux/debian/", "Debian" - ) + ret = s3_upload( + utils.glob_path("deb/*.deb"), + "desktop/linux/debian/") utils.set_summary("desktop deb deploy", ret) if "deb-astra" in branding.desktop_make_targets: utils.log_h2("desktop deb-astra deploy") - ret = aws_s3_upload( - utils.glob_path("deb-astra/*.deb"), - "linux/astra/", "Astra Linux Special Edition" - ) + ret = s3_upload( + utils.glob_path("deb-astra/*.deb"), + "desktop/linux/astra/") utils.set_summary("desktop deb-astra deploy", ret) if "rpm" in branding.desktop_make_targets: utils.log_h2("desktop rpm deploy") - ret = aws_s3_upload( - utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ - + utils.glob_path("rpm/builddir/RPMS/noarch/*.rpm"), - "linux/rhel/", "CentOS" - ) + ret = s3_upload( + utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), + "desktop/linux/rhel/") utils.set_summary("desktop rpm deploy", ret) if "suse-rpm" in branding.desktop_make_targets: utils.log_h2("desktop suse-rpm deploy") - ret = aws_s3_upload( - utils.glob_path("suse-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ - + utils.glob_path("suse-rpm/builddir/RPMS/noarch/*.rpm"), - "linux/suse/", "SUSE Linux" - ) + ret = s3_upload( + utils.glob_path("suse-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), + "desktop/linux/suse/") utils.set_summary("desktop suse-rpm deploy", ret) if "apt-rpm" in branding.desktop_make_targets: utils.log_h2("desktop apt-rpm deploy") - ret = aws_s3_upload( - utils.glob_path("apt-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ - + utils.glob_path("apt-rpm/builddir/RPMS/noarch/*.rpm"), - "linux/altlinux/", "ALT Linux" - ) + ret = s3_upload( + utils.glob_path("apt-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), + "desktop/linux/altlinux/") utils.set_summary("desktop apt-rpm deploy", ret) if "urpmi" in branding.desktop_make_targets: utils.log_h2("desktop urpmi deploy") - ret = aws_s3_upload( - utils.glob_path("urpmi/builddir/RPMS/" + rpm_arch + "/*.rpm") \ - + utils.glob_path("urpmi/builddir/RPMS/noarch/*.rpm"), - "linux/rosa/", "ROSA" - ) + ret = s3_upload( + utils.glob_path("urpmi/builddir/RPMS/" + rpm_arch + "/*.rpm"), + "desktop/linux/rosa/") utils.set_summary("desktop urpmi deploy", ret) else: if "tar" in branding.desktop_make_targets: diff --git a/scripts/package_mobile.py b/scripts/package_mobile.py index 967dcee..fe35720 100644 --- a/scripts/package_mobile.py +++ b/scripts/package_mobile.py @@ -15,31 +15,28 @@ def make(): def make_mobile(): utils.set_cwd("build_tools/out") + zip_file = "build-" + common.version + "-" + common.build + ".zip" + if common.clean: utils.log_h2("mobile clean") utils.sh("rm -rfv *.zip", verbose=True) - zip_file = "build-" + common.version + "-" + common.build + ".zip" - s3_key = "mobile/android/" + zip_file - utils.log_h2("mobile build") ret = utils.sh("zip -r " + zip_file + " ./android* ./js", verbose=True) utils.set_summary("mobile build", ret) if common.deploy: - utils.log_h2("mobile deploy") - if not utils.is_file(zip_file): - utils.log_err("file not exist: " + zip_file) - ret = False - elif ret: - ret = utils.sh( - "aws s3 cp --acl public-read --no-progress " \ - + "--metadata md5=" + utils.get_md5(zip_file) + " " \ - + zip_file + " s3://" + branding.s3_bucket + "/" + s3_key, - verbose=True - ) if ret: - utils.add_deploy_data("mobile", "Android", zip_file, s3_key) + utils.log_h2("mobile deploy") + key = "mobile/android/" + zip_file + aws_kwargs = { "acl": "public-read" } + if hasattr(branding, "s3_endpoint_url"): + aws_kwargs["endpoint_url"] = branding.s3_endpoint_url + ret = utils.s3_upload( + zip_file, "s3://" + branding.s3_bucket + "/" + key, **aws_kwargs) + if ret: + utils.add_deploy_data(key) + utils.log("URL: " + branding.s3_base_url + "/" + key) utils.set_summary("mobile deploy", ret) utils.set_cwd(common.workspace_dir) diff --git a/scripts/package_server.py b/scripts/package_server.py index e0229cd..a98434a 100644 --- a/scripts/package_server.py +++ b/scripts/package_server.py @@ -14,33 +14,20 @@ def make(edition): utils.log("Unsupported host OS") return -def aws_s3_upload(files, key, edition, ptype=None): - if not files: - return False +def s3_upload(files, dst): + if not files: return False ret = True - key = "server/" + key - for file in files: - if not utils.is_file(file): - utils.log_err("file not exist: " + file) - ret &= False - continue - args = ["aws"] + for f in files: + key = dst + utils.get_basename(f) if dst.endswith("/") else dst + aws_kwargs = { "acl": "public-read" } if hasattr(branding, "s3_endpoint_url"): - args += ["--endpoint-url=" + branding.s3_endpoint_url] - args += [ - "s3", "cp", "--no-progress", "--acl", "public-read", - "--metadata", "md5=" + utils.get_md5(file), - file, "s3://" + branding.s3_bucket + "/" + key - ] - if common.os_family == "windows": - upload = utils.cmd(*args, verbose=True) - else: - upload = utils.sh(" ".join(args), verbose=True) + aws_kwargs["endpoint_url"] = branding.s3_endpoint_url + upload = utils.s3_upload( + f, "s3://" + branding.s3_bucket + "/" + key, **aws_kwargs) + if upload: + utils.add_deploy_data(key) + utils.log("URL: " + branding.s3_base_url + "/" + key) ret &= upload - if upload and ptype is not None: - full_key = key - if full_key.endswith("/"): full_key += utils.get_basename(file) - utils.add_deploy_data("server_" + edition, ptype, file, full_key) return ret def make_windows(edition): @@ -62,10 +49,7 @@ def make_windows(edition): if common.deploy and ret: utils.log_h2("server " + edition + " inno deploy") - ret = aws_s3_upload( - utils.glob_path("exe/*.exe"), - "win/inno/", edition, "Installer" - ) + ret = s3_upload(utils.glob_path("exe/*.exe"), "server/win/inno/") utils.set_summary("server " + edition + " inno deploy", ret) utils.set_cwd(common.workspace_dir) @@ -89,39 +73,31 @@ def make_linux(edition): ret = utils.sh("make clean && make " + " ".join(make_args), verbose=True) utils.set_summary("server " + edition + " build", ret) - rpm_arch = "x86_64" - if common.platform == "linux_aarch64": rpm_arch = "aarch64" - if common.deploy: - utils.log_h2("server " + edition + " deploy") if ret: if "deb" in branding.server_make_targets: utils.log_h2("server " + edition + " deb deploy") - ret = aws_s3_upload( - utils.glob_path("deb/*.deb"), - "linux/debian/", edition, "Debian" - ) + ret = s3_upload( + utils.glob_path("deb/*.deb"), + "server/linux/debian/") utils.set_summary("server " + edition + " deb deploy", ret) if "rpm" in branding.server_make_targets: utils.log_h2("server " + edition + " rpm deploy") - ret = aws_s3_upload( - utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), - "linux/rhel/", edition, "CentOS" - ) + ret = s3_upload( + utils.glob_path("rpm/builddir/RPMS/*/*.rpm"), + "server/linux/rhel/") utils.set_summary("server " + edition + " rpm deploy", ret) if "apt-rpm" in branding.server_make_targets: utils.log_h2("server " + edition + " apt-rpm deploy") - ret = aws_s3_upload( - utils.glob_path("apt-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), - "linux/altlinux/", edition, "ALT Linux" - ) + ret = s3_upload( + utils.glob_path("apt-rpm/builddir/RPMS/*/*.rpm"), + "server/linux/altlinux/") utils.set_summary("server " + edition + " apt-rpm deploy", ret) if "tar" in branding.server_make_targets: utils.log_h2("server " + edition + " snap deploy") - ret = aws_s3_upload( - utils.glob_path("*.tar.gz"), - "linux/generic/", edition, "Snap" - ) + ret = s3_upload( + utils.glob_path("*.tar.gz"), + "server/linux/snap/") utils.set_summary("server " + edition + " snap deploy", ret) else: if "deb" in branding.server_make_targets: diff --git a/scripts/package_utils.py b/scripts/package_utils.py index d373ca9..a51d628 100644 --- a/scripts/package_utils.py +++ b/scripts/package_utils.py @@ -4,7 +4,6 @@ import codecs import glob import hashlib -import json import os import platform import re @@ -107,11 +106,25 @@ def glob_file(path): return glob.glob(path)[0] return -def get_md5(path): +def get_hash_sha256(path): if os.path.exists(path): - md5_hash = hashlib.md5() - md5_hash.update(open(path, "rb").read()) - return md5_hash.hexdigest() + h = hashlib.sha256() + h.update(open(path, "rb").read()) + return h.hexdigest() + return + +def get_hash_sha1(path): + if os.path.exists(path): + h = hashlib.sha1() + h.update(open(path, "rb").read()) + return h.hexdigest() + return + +def get_hash_md5(path): + if os.path.exists(path): + h = hashlib.md5() + h.update(open(path, "rb").read()) + return h.hexdigest() return def create_dir(path, verbose=True): @@ -258,18 +271,9 @@ def set_summary(target, status): common.summary.append({target: status}) return -def add_deploy_data(product, ptype, src, dst): - common.deploy_data.append({ - "platform": common.platformTitles[common.platform], - "product": product, - "type": ptype, - # "local": get_path(src), - "size": get_file_size(get_path(src)), - "key": dst - }) - file = open(get_path(common.workspace_dir + "/deploy.json"), 'w') - file.write(json.dumps(common.deploy_data, sort_keys=True, indent=4)) - file.close() +def add_deploy_data(key): + with open(common.deploy_data, 'a+') as f: + f.write(key + "\n") return def cmd(*args, **kwargs): @@ -334,7 +338,7 @@ def download_file(url, path, md5, verbose=False): log(" path: " + url) log(" md5: " + md5) if is_file(path): - if get_md5(path) == md5: + if get_hash_md5(path) == md5: log_err("file already exist (match checksum)") return True else: @@ -344,7 +348,7 @@ def download_file(url, path, md5, verbose=False): "(New-Object System.Net.WebClient).DownloadFile('%s','%s')" % (url, path), verbose=True ) - md5_new = get_md5(path) + md5_new = get_hash_md5(path) if md5 != md5_new: log_err("checksum didn't match (%s != %s)" % (md5, md5_new)) return False @@ -363,18 +367,63 @@ def sh(command, **kwargs): return False if kwargs.get("chdir") and is_dir(kwargs["chdir"]): oldcwd = get_cwd() - set_cwd(kwargs["chdir"]) + set_cwd(kwargs["chdir"], verbose=False) ret = subprocess.call( command, stderr=subprocess.STDOUT, shell=True ) == 0 if kwargs.get("chdir") and oldcwd: - set_cwd(oldcwd) + set_cwd(oldcwd, verbose=False) return ret def sh_output(command, **kwargs): if kwargs.get("verbose"): log("- sh_output:") log(" command: " + command) - return subprocess.check_output( + if kwargs.get("chdir"): + log(" chdir: " + kwargs["chdir"]) + if kwargs.get("chdir") and is_dir(kwargs["chdir"]): + oldcwd = get_cwd() + set_cwd(kwargs["chdir"], verbose=False) + ret = subprocess.check_output( command, stderr=subprocess.STDOUT, shell=True ).decode("utf-8") + log(ret) + if kwargs.get("chdir") and oldcwd: + set_cwd(oldcwd, verbose=False) + return ret + +def s3_upload(src, dst, **kwargs): + if not is_file(src): + log_err("file not exist: " + src) + return False + metadata = "sha256=" + get_hash_sha256(src) \ + + ",sha1=" + get_hash_sha1(src) \ + + ",md5=" + get_hash_md5(src) + args = ["aws"] + if kwargs.get("endpoint_url"): + args += ["--endpoint-url", kwargs["endpoint_url"]] + args += ["s3", "cp", "--no-progress"] + if kwargs.get("acl"): + args += ["--acl", kwargs["acl"]] + args += ["--metadata", metadata, src, dst] + if is_windows(): + ret = cmd(*args, verbose=True) + else: + ret = sh(" ".join(args), verbose=True) + return ret + +def s3_sync(src, dst, **kwargs): + args = ["aws"] + if kwargs.get("endpoint_url"): + args += ["--endpoint-url", kwargs["endpoint_url"]] + args += ["s3", "sync", "--no-progress"] + if kwargs.get("acl"): + args += ["--acl", kwargs["acl"]] + if kwargs.get("delete") and kwargs["delete"]: + args += ["--delete"] + args += [src, dst] + if is_windows(): + ret = cmd(*args, verbose=True) + else: + ret = sh(" ".join(args), verbose=True) + return ret From 5012e4e9bd302c44d3a35eff40886ef207843086 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 8 Aug 2023 19:24:19 +0300 Subject: [PATCH 5/5] [develop] Use npm ci instead of npm i when building web-apps --- scripts/build_js.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_js.py b/scripts/build_js.py index 7c22c69..60f54e7 100644 --- a/scripts/build_js.py +++ b/scripts/build_js.py @@ -146,15 +146,15 @@ def build_js_develop(root_dir): _run_npm_ci(root_dir + external_folder + "/sdkjs/build") _run_grunt(root_dir + external_folder + "/sdkjs/build", get_build_param(False) + base.sdkjs_addons_param()) _run_grunt(root_dir + external_folder + "/sdkjs/build", ["develop"] + base.sdkjs_addons_param()) - _run_npm(root_dir + external_folder + "/web-apps/build") - _run_npm(root_dir + external_folder + "/web-apps/build/sprites") + _run_npm_ci(root_dir + external_folder + "/web-apps/build") + _run_npm_ci(root_dir + external_folder + "/web-apps/build/sprites") _run_grunt(root_dir + external_folder + "/web-apps/build/sprites", []) old_cur = os.getcwd() old_product_version = base.get_env("PRODUCT_VERSION") base.set_env("PRODUCT_VERSION", old_product_version + "d") os.chdir(root_dir + external_folder + "/web-apps/vendor/framework7-react") - base.cmd("npm", ["install"]) + base.cmd("npm", ["ci"]) base.cmd("npm", ["run", "deploy-word"]) base.cmd("npm", ["run", "deploy-cell"]) base.cmd("npm", ["run", "deploy-slide"])