From dd9a8b9df52f5a09b76b8f802404d6214cfac572 Mon Sep 17 00:00:00 2001 From: Semyon Bezrukov Date: Fri, 23 Dec 2022 20:11:12 +0300 Subject: [PATCH] Fix develop (#588) * Fix msi build (#583) * Add packages upload s3 endpoint url option (#584) * Fix package build (#585) * Refactoring package scripts (#587) * Refactoring logs * Refactoring packages * Small fix --- scripts/package_builder.py | 131 ++++++++-------- scripts/package_core.py | 12 +- scripts/package_desktop.py | 310 ++++++++++++++++++++----------------- scripts/package_mobile.py | 27 ++-- scripts/package_server.py | 130 ++++++++-------- scripts/package_utils.py | 133 ++++++++-------- 6 files changed, 386 insertions(+), 357 deletions(-) diff --git a/scripts/package_builder.py b/scripts/package_builder.py index da5d006..74ff548 100644 --- a/scripts/package_builder.py +++ b/scripts/package_builder.py @@ -17,23 +17,26 @@ def make(): def aws_s3_upload(files, key, ptype=None): key = "builder/" + key - rc = 0 for file in files: + args = ["aws"] + if hasattr(branding, "s3_endpoint_url"): + args += ["--endpoint-url=" + branding.s3_endpoint_url] + args += [ + "s3", "cp", "--no-progress", "--acl", "public-read", + file, "s3://" + branding.s3_bucket + "/" + key + ] if common.os_family == "windows": - rc += utils.cmd( - "aws", "s3", "cp", "--acl", "public-read", "--no-progress", - file, "s3://" + branding.s3_bucket + "/" + key, - verbose=True) + ret = utils.cmd(*args, verbose=True) else: - rc += utils.sh( - "aws s3 cp --acl public-read --no-progress " \ - + file + " s3://" + branding.s3_bucket + "/" + key, - verbose=True) - if rc == 0 and ptype is not None: + ret = utils.sh(" ".join(args), verbose=True) + if ret and ptype is not None: full_key = key if full_key.endswith("/"): full_key += utils.get_basename(file) - utils.add_deploy_data("builder", ptype, file, full_key, branding.s3_bucket, branding.s3_region) - return rc + utils.add_deploy_data( + "builder", ptype, file, full_key, + branding.s3_bucket, branding.s3_region + ) + return ret def make_windows(): global inno_file, zip_file, suffix, key_prefix @@ -58,13 +61,11 @@ def make_windows(): utils.log_h2("builder clean") utils.delete_dir("build") - utils.log_h1("copy arifacts") + utils.log_h2("copy arifacts") utils.create_dir("build\\app") utils.copy_dir_content(source_dir, "build\\app\\") - # if "builder-zip" in common.targets: make_zip() - # if "builder-inno" in common.targets: make_inno() utils.set_cwd(common.workspace_dir) @@ -72,23 +73,24 @@ def make_windows(): def make_zip(): utils.log_h2("builder zip build") - utils.log_h2(zip_file) - rc = utils.cmd("7z", "a", "-y", zip_file, ".\\app\\*", - chdir="build", creates="build\\" + zip_file, verbose=True) - utils.set_summary("builder zip build", rc == 0) + utils.log_h3(zip_file) - if rc == 0: + ret = utils.cmd("7z", "a", "-y", zip_file, ".\\app\\*", + chdir="build", creates="build\\" + zip_file, verbose=True) + utils.set_summary("builder zip build", ret) + + if common.deploy and ret: utils.log_h2("builder zip deploy") - rc = aws_s3_upload( - ["build\\" + zip_file], - "win/generic/%s/" % common.channel, - "Portable") - utils.set_summary("builder zip deploy", rc == 0) + ret = aws_s3_upload( + ["build\\" + zip_file], "win/generic/%s/" % common.channel, "Portable" + ) + utils.set_summary("builder zip deploy", ret) return def make_inno(): utils.log_h2("builder inno build") - utils.log_h2(inno_file) + utils.log_h3(inno_file) + args = [ "-Arch " + suffix, "-Version " + common.version, @@ -99,64 +101,63 @@ def make_inno(): if common.sign: args.append("-Sign") args.append("-CertName '%s'" % branding.cert_name) - rc = utils.ps1(".\\make_inno.ps1", args, - creates="build\\" + inno_file, verbose=True) - utils.set_summary("builder inno build", rc == 0) + ret = utils.ps1( + ".\\make_inno.ps1", args, creates="build\\" + inno_file, verbose=True + ) + utils.set_summary("builder inno build", ret) - if rc == 0: + if common.deploy and ret: utils.log_h2("builder inno deploy") - rc = aws_s3_upload( - ["build\\" + inno_file], - "win/inno/%s/" % common.channel, - "Installer") - utils.set_summary("builder inno deploy", rc == 0) + ret = aws_s3_upload( + ["build\\" + inno_file], "win/inno/%s/" % common.channel, "Installer" + ) + utils.set_summary("builder inno deploy", ret) return def make_linux(): utils.set_cwd("document-builder-package") - utils.log_h2("builder clean") - rc = utils.sh("make clean", verbose=True) - utils.set_summary("builder clean", rc == 0) - utils.log_h2("builder build") args = [] if common.platform == "linux_aarch64": args += ["-e", "UNAME_M=aarch64"] if not branding.onlyoffice: args += ["-e", "BRANDING_DIR=../" + common.branding + "/document-builder-package"] - rc = utils.sh("make packages " + " ".join(args), verbose=True) - utils.set_summary("builder build", rc == 0) + ret = utils.sh("make clean && make packages " + " ".join(args), verbose=True) + utils.set_summary("builder build", ret) rpm_arch = "x86_64" if common.platform == "linux_aarch64": rpm_arch = "aarch64" - if rc == 0: - # utils.log_h2("builder tar deploy") - # rc = aws_s3_upload( - # utils.glob_path("tar/*.tar.gz"), - # "linux/generic/%s/" % common.channel, - # "Portable") - # utils.set_summary("builder tar deploy", rc == 0) + if common.deploy: + utils.log_h2("builder deploy") + if ret: + # utils.log_h2("builder tar deploy") + # ret = aws_s3_upload( + # utils.glob_path("tar/*.tar.gz"), + # "linux/generic/%s/" % common.channel, + # "Portable") + # utils.set_summary("builder tar deploy", ret) - utils.log_h2("builder deb deploy") - rc = aws_s3_upload( - utils.glob_path("deb/*.deb"), - "linux/debian/%s/" % common.channel, - "Debian") - utils.set_summary("builder deb deploy", rc == 0) + utils.log_h2("builder deb deploy") + ret = aws_s3_upload( + utils.glob_path("deb/*.deb"), + "linux/debian/%s/" % common.channel, + "Debian" + ) + utils.set_summary("builder deb deploy", ret) - utils.log_h2("builder rpm deploy") - rc = aws_s3_upload( - utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), - "linux/rhel/%s/" % common.channel, - "CentOS") - utils.set_summary("builder rpm deploy", rc == 0) - - else: - # utils.set_summary("builder tar deploy", False) - utils.set_summary("builder deb deploy", False) - utils.set_summary("builder rpm deploy", False) + utils.log_h2("builder rpm deploy") + ret = aws_s3_upload( + utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), + "linux/rhel/%s/" % common.channel, + "CentOS" + ) + utils.set_summary("builder rpm deploy", ret) + else: + # utils.set_summary("builder tar deploy", False) + utils.set_summary("builder deb deploy", False) + utils.set_summary("builder rpm deploy", False) utils.set_cwd(common.workspace_dir) return diff --git a/scripts/package_core.py b/scripts/package_core.py index 51310d8..eb25899 100644 --- a/scripts/package_core.py +++ b/scripts/package_core.py @@ -42,18 +42,18 @@ def make_core(): args = ["aws", "s3", "cp", "--acl", "public-read", "--no-progress", core_7z, "s3://" + branding.s3_bucket + "/" + dest_version + "core.7z"] if common.os_family == "windows": - rc = utils.cmd(*args, verbose=True) + ret = utils.cmd(*args, verbose=True) else: - rc = utils.sh(" ".join(args), verbose=True) - if rc == 0: + ret = utils.sh(" ".join(args), verbose=True) + if ret: utils.add_deploy_data("core", "Archive", core_7z, dest_version + "core.7z", branding.s3_bucket, branding.s3_region) args = ["aws", "s3", "sync", "--delete", "--acl", "public-read", "--no-progress", "s3://" + branding.s3_bucket + "/" + dest_version, "s3://" + branding.s3_bucket + "/" + dest_latest] if common.os_family == "windows": - rc = utils.cmd(*args, verbose=True) + ret &= utils.cmd(*args, verbose=True) else: - rc = utils.sh(" ".join(args), verbose=True) - utils.set_summary("core deploy", rc == 0) + ret &= utils.sh(" ".join(args), verbose=True) + utils.set_summary("core deploy", ret) return diff --git a/scripts/package_desktop.py b/scripts/package_desktop.py index 422b665..5338b6a 100644 --- a/scripts/package_desktop.py +++ b/scripts/package_desktop.py @@ -20,19 +20,19 @@ def make(): def aws_s3_upload(files, key, ptype=None): key = "desktop/" + key - rc = 0 for file in files: + args = ["aws"] + if hasattr(branding, "s3_endpoint_url"): + args += ["--endpoint-url=" + branding.s3_endpoint_url] + args += [ + "s3", "cp", "--no-progress", "--acl", "public-read", + file, "s3://" + branding.s3_bucket + "/" + key + ] if common.os_family == "windows": - rc += utils.cmd( - "aws", "s3", "cp", "--acl", "public-read", "--no-progress", - file, "s3://" + branding.s3_bucket + "/" + key, - verbose=True) + ret = utils.cmd(*args, verbose=True) else: - rc += utils.sh( - "aws s3 cp --acl public-read --no-progress " \ - + file + " s3://" + branding.s3_bucket + "/" + key, - verbose=True) - if rc == 0 and ptype is not None: + ret = utils.sh(" ".join(args), verbose=True) + if ret and ptype is not None: full_key = key if full_key.endswith("/"): full_key += utils.get_basename(file) utils.add_deploy_data("desktop", ptype, file, full_key, branding.s3_bucket, branding.s3_region) @@ -116,20 +116,20 @@ def make_windows(): def make_zip(): utils.log_h2("desktop zip build") - rc = utils.cmd( + ret = utils.cmd( "7z", "a", "-y", zip_file, source_dir + "\\*", - creates=zip_file, - verbose=True + creates=zip_file, verbose=True ) - utils.set_summary("desktop zip build", rc == 0) + utils.set_summary("desktop zip build", ret) - if rc == 0: + if common.deploy and ret: utils.log_h2("desktop zip deploy") - rc = aws_s3_upload( + ret = aws_s3_upload( [zip_file], "win/generic/%s/" % common.channel, - "Portable") - utils.set_summary("desktop zip deploy", rc == 0) + "Portable" + ) + utils.set_summary("desktop zip deploy", ret) return def download_vcredist(year): @@ -142,17 +142,17 @@ def download_vcredist(year): utils.log_h2(vcredist_file) utils.create_dir(utils.get_dirname(vcredist_file)) - rc = utils.download_file(link, vcredist_file, md5, verbose=True) - utils.set_summary("vcredist " + year + " download", rc == 0) - return rc == 0 + ret = utils.download_file(link, vcredist_file, md5, verbose=True) + utils.set_summary("vcredist " + year + " download", ret) + return ret def make_inno(): global iscc_args utils.log_h2("desktop inno build") - utils.log_h2(inno_file) + utils.log_h3(inno_file) iscc_args = [ - # "/Qp", + "/Qp", "/DsAppVersion=" + package_version, "/DDEPLOY_PATH=" + source_dir, "/D_ARCH=" + inno_arch_list[common.platform] @@ -169,21 +169,22 @@ def make_inno(): iscc_args.append("/Sbyparam=signtool.exe sign /v /n $q" + \ branding.cert_name + "$q /t " + common.tsa_server + " $f") args = ["iscc"] + iscc_args + ["common.iss"] - rc = utils.cmd(*args, creates=inno_file, verbose=True) - utils.set_summary("desktop inno build", rc == 0) + ret = utils.cmd(*args, creates=inno_file, verbose=True) + utils.set_summary("desktop inno build", ret) - if rc == 0: + if common.deploy and ret: utils.log_h2("desktop inno deploy") - rc = aws_s3_upload( + ret = aws_s3_upload( [inno_file], "win/inno/%s/%s/" % (common.version, common.build), - "Installer") - utils.set_summary("desktop inno deploy", rc == 0) + "Installer" + ) + utils.set_summary("desktop inno deploy", ret) return def make_inno_help(): utils.log_h2("desktop inno help build") - utils.log_h2(inno_help_file) + utils.log_h3(inno_help_file) args = [ "iscc", @@ -202,16 +203,17 @@ def make_inno_help(): args.append("/Sbyparam=signtool.exe sign /v /n $q" + \ branding.cert_name + "$q /t " + common.tsa_server + " $f") args.append("help.iss") - rc = utils.cmd(*args, creates=inno_help_file, verbose=True) - utils.set_summary("desktop inno help build", rc == 0) + ret = utils.cmd(*args, creates=inno_help_file, verbose=True) + utils.set_summary("desktop inno help build", ret) - if rc == 0: + if common.deploy and ret: utils.log_h2("desktop inno help deploy") - rc = aws_s3_upload( + ret = aws_s3_upload( [inno_help_file], "win/inno/%s/%s/" % (common.version, common.build), - "Installer") - utils.set_summary("desktop inno help deploy", rc == 0) + "Installer" + ) + utils.set_summary("desktop inno help deploy", ret) return def make_inno_update(): @@ -219,16 +221,17 @@ def make_inno_update(): utils.log_h2(inno_update_file) args = ["iscc"] + iscc_args + ["/DTARGET_NAME=" + inno_file, "update_common.iss"] - rc = utils.cmd(*args, creates=inno_update_file, verbose=True) - utils.set_summary("desktop inno update build", rc == 0) + ret = utils.cmd(*args, creates=inno_update_file, verbose=True) + utils.set_summary("desktop inno update build", ret) - if rc == 0: + if common.deploy and ret: utils.log_h2("desktop inno update deploy") - rc = aws_s3_upload( + ret = aws_s3_upload( [inno_update_file], "win/inno/%s/%s/" % (common.version, common.build), - "WinSparkle") - utils.set_summary("desktop inno update deploy", rc == 0) + "WinSparkle" + ) + utils.set_summary("desktop inno update deploy", ret) return def make_winsparkle_files(): @@ -248,14 +251,14 @@ def make_winsparkle_files(): ] appcast = "update/appcast.xml" - utils.log_h2(appcast) + utils.log_h3(appcast) args = ["env", "LANG=en_US.UTF-8", "awk", "-v", "Prod=1"] + \ awk_args + ["-f", "update/appcast.xml.awk"] appcast_result = utils.cmd_output(*args, verbose=True) utils.write_file(appcast, appcast_result) appcast_test = "update/appcast-test.xml" - utils.log_h2(appcast_test) + utils.log_h3(appcast_test) args = ["env", "LANG=en_US.UTF-8", "awk"] + \ awk_args + ["-f", "update/appcast.xml.awk"] appcast_result = utils.cmd_output(*args, verbose=True) @@ -270,7 +273,7 @@ def make_winsparkle_files(): changes = "update/%s.html" % base if lang == "en": encoding = "en_US.UTF-8" elif lang == "ru": encoding = "ru_RU.UTF-8" - utils.log_h2(changes) + utils.log_h3(changes) changes_file = "%s/%s.html" % (changes_dir, lang) args = ["env", "LANG=" + encoding, "awk"] + awk_args + \ ["-f", "update/changes.html.awk", changes_file] @@ -282,17 +285,19 @@ def make_winsparkle_files(): else: utils.log("! file not exist: " + changes_file) - utils.log_h2("desktop winsparkle files deploy") - rc = aws_s3_upload( - utils.glob_path("update/*.xml") + utils.glob_path("update/*.html"), - "win/inno/%s/%s/" % (common.version, common.build), - "WinSparkle") - utils.set_summary("desktop winsparkle files deploy", rc == 0) + if common.deploy: + utils.log_h2("desktop winsparkle files deploy") + ret = aws_s3_upload( + utils.glob_path("update/*.xml") + utils.glob_path("update/*.html"), + "win/inno/%s/%s/" % (common.version, common.build), + "WinSparkle" + ) + utils.set_summary("desktop winsparkle files deploy", ret) return def make_advinst(): utils.log_h2("desktop advinst build") - utils.log_h2(advinst_file) + utils.log_h3(advinst_file) arch = arch_list[common.platform] @@ -360,7 +365,7 @@ def make_advinst(): "NewSync CUSTOM_PATH " + source_dir + "\\..\\MediaViewer", "UpdateFile CUSTOM_PATH\\ImageViewer.exe " + source_dir + "\\..\\MediaViewer\\ImageViewer.exe", "UpdateFile CUSTOM_PATH\\VideoPlayer.exe " + source_dir + "\\..\\MediaViewer\\VideoPlayer.exe", - "SetProperty ASCC_REG_PREFIX=" + ascc_reg_prefix + "SetProperty ASCC_REG_PREFIX=" + branding.ascc_reg_prefix ] aic_content += [ "AddOsLc -buildname DefaultBuild -arch " + arch, @@ -372,17 +377,18 @@ def make_advinst(): "Rebuild -buildslist DefaultBuild" ] utils.write_file("DesktopEditors.aic", "\r\n".join(aic_content), "utf-8-sig") - rc = utils.cmd("AdvancedInstaller.com", "/execute", \ + ret = utils.cmd("AdvancedInstaller.com", "/execute", \ "DesktopEditors.aip", "DesktopEditors.aic", verbose=True) - utils.set_summary("desktop advinst build", rc == 0) + utils.set_summary("desktop advinst build", ret) - if rc == 0: + if common.deploy and ret: utils.log_h2("desktop advinst deploy") - rc = aws_s3_upload( + ret = aws_s3_upload( [advinst_file], "win/advinst/%s/" % common.channel, - "Installer") - utils.set_summary("desktop advinst deploy", rc == 0) + "Installer" + ) + utils.set_summary("desktop advinst deploy", ret) return # @@ -448,21 +454,22 @@ def make_macos(): def make_dmg(): utils.log_h2("desktop dmg build") - utils.log_h2(scheme) - utils.log_h2("build/" + package_name + ".app") - rc = utils.sh( + utils.log_h3(scheme) + utils.log_h3("build/" + package_name + ".app") + ret = utils.sh( "bundler exec fastlane " + lane + " skip_git_bump:true", verbose=True ) - utils.set_summary("desktop dmg build", rc == 0) + utils.set_summary("desktop dmg build", ret) - if rc == 0: + if common.deploy and ret: utils.log_h2("desktop dmg deploy") - rc = aws_s3_upload( + ret = aws_s3_upload( utils.glob_path("build/*.dmg"), "mac/%s/%s/%s/" % (suffix, common.version, common.build), - "Disk Image") - utils.set_summary("desktop dmg deploy", rc == 0) + "Disk Image" + ) + utils.set_summary("desktop dmg deploy", ret) return def make_sparkle_updates(): @@ -480,7 +487,7 @@ def make_sparkle_updates(): utils.copy_file(macos_zip, updates_dir) if "en" in update_changes_list: - notes_src = "%s/%s/%s.html" % (changes_dir, app_version, update_changes_list["en"]) + notes_sret = "%s/%s/%s.html" % (changes_dir, app_version, update_changes_list["en"]) notes_dst = "%s/%s.html" % (updates_dir, zip_filename) if utils.is_file(notes_src): utils.copy_file(notes_src, notes_dst) @@ -492,7 +499,7 @@ def make_sparkle_updates(): utils.write_file(notes_dst, '\n') if "ru" in update_changes_list: - notes_src = "%s/%s/%s.html" % (changes_dir, app_version, update_changes_list["ru"]) + notes_sret = "%s/%s/%s.html" % (changes_dir, app_version, update_changes_list["ru"]) if update_changes_list["ru"] != "ReleaseNotes": notes_dst = "%s/%s.ru.html" % (updates_dir, zip_filename) else: @@ -538,22 +545,25 @@ def make_sparkle_updates(): utils.sh("md5 *.zip *.delta > md5sums.txt", chdir="build/update", verbose=True) utils.sh("shasum -a 256 *.zip *.delta > sha256sums.txt", chdir="build/update", verbose=True) - utils.log_h2("desktop sparkle files deploy") - rc = aws_s3_upload( - [macos_zip] \ - + utils.glob_path("build/update/*.delta") \ - + utils.glob_path("build/update/*.xml") \ - + utils.glob_path("build/update/*.html"), - "mac/%s/%s/%s/" % (suffix, common.version, common.build), - "Sparkle") - utils.set_summary("desktop sparkle files deploy", rc == 0) + if common.deploy: + utils.log_h2("desktop sparkle files deploy") + ret = aws_s3_upload( + [macos_zip] \ + + utils.glob_path("build/update/*.delta") \ + + utils.glob_path("build/update/*.xml") \ + + utils.glob_path("build/update/*.html"), + "mac/%s/%s/%s/" % (suffix, common.version, common.build), + "Sparkle" + ) + utils.set_summary("desktop sparkle files deploy", ret) - utils.log_h2("desktop checksums deploy") - rc = aws_s3_upload( - utils.glob_path("build/update/*.txt"), - "mac/%s/%s/%s/" % (suffix, common.version, common.build), - "Checksums") - utils.set_summary("desktop checksums deploy", rc == 0) + utils.log_h2("desktop checksums deploy") + ret = aws_s3_upload( + utils.glob_path("build/update/*.txt"), + "mac/%s/%s/%s/" % (suffix, common.version, common.build), + "Checksums" + ) + utils.set_summary("desktop checksums deploy", ret) return # @@ -563,84 +573,92 @@ def make_sparkle_updates(): def make_linux(): utils.set_cwd("desktop-apps/win-linux/package/linux") - rc = utils.sh("make clean", verbose=True) - utils.set_summary("desktop clean", rc == 0) + ret = utils.sh("make clean", verbose=True) + utils.set_summary("desktop clean", ret) args = [] if common.platform == "linux_aarch64": args += ["-e", "UNAME_M=aarch64"] if not branding.onlyoffice: args += ["-e", "BRANDING_DIR=../../../../" + common.branding + "/desktop-apps/win-linux/package/linux"] - rc = utils.sh("make packages " + " ".join(args), verbose=True) - utils.set_summary("desktop build", rc == 0) + ret = utils.sh("make packages " + " ".join(args), verbose=True) + utils.set_summary("desktop build", ret) rpm_arch = "x86_64" if common.platform == "linux_aarch64": rpm_arch = "aarch64" - if rc == 0: - utils.log_h2("desktop tar deploy") - rc = aws_s3_upload( - utils.glob_path("tar/*.tar.gz") + utils.glob_path("tar/*.tar.xz"), - "linux/generic/%s/" % common.channel, - "Portable") - utils.set_summary("desktop tar deploy", rc == 0) + if common.deploy: + utils.log_h2("desktop deploy") + if ret: + utils.log_h2("desktop tar deploy") + ret = aws_s3_upload( + utils.glob_path("tar/*.tar.gz") + utils.glob_path("tar/*.tar.xz"), + "linux/generic/%s/" % common.channel, + "Portable" + ) + utils.set_summary("desktop tar deploy", ret) - utils.log_h2("desktop deb deploy") - rc = aws_s3_upload( - utils.glob_path("deb/*.deb"), - "linux/debian/%s/" % common.channel, - "Debian") - utils.set_summary("desktop deb deploy", rc == 0) + utils.log_h2("desktop deb deploy") + ret = aws_s3_upload( + utils.glob_path("deb/*.deb"), + "linux/debian/%s/" % common.channel, + "Debian" + ) + utils.set_summary("desktop deb deploy", ret) - utils.log_h2("desktop rpm deploy") - rc = aws_s3_upload( - utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ - + utils.glob_path("rpm/builddir/RPMS/noarch/*.rpm"), - "linux/rhel/%s/" % common.channel, - "CentOS") - utils.set_summary("desktop rpm deploy", rc == 0) + utils.log_h2("desktop rpm deploy") + ret = aws_s3_upload( + utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ + + utils.glob_path("rpm/builddir/RPMS/noarch/*.rpm"), + "linux/rhel/%s/" % common.channel, + "CentOS" + ) + utils.set_summary("desktop rpm deploy", ret) - utils.log_h2("desktop rpm-apt deploy") - rc = aws_s3_upload( - utils.glob_path("apt-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ - + utils.glob_path("apt-rpm/builddir/RPMS/noarch/*.rpm"), - "linux/altlinux/%s/" % common.channel, - "ALT Linux") - utils.set_summary("desktop rpm-apt deploy", rc == 0) + utils.log_h2("desktop rpm-apt deploy") + ret = aws_s3_upload( + utils.glob_path("apt-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ + + utils.glob_path("apt-rpm/builddir/RPMS/noarch/*.rpm"), + "linux/altlinux/%s/" % common.channel, + "ALT Linux" + ) + utils.set_summary("desktop rpm-apt deploy", ret) - utils.log_h2("desktop urpmi deploy") - rc = aws_s3_upload( - utils.glob_path("urpmi/builddir/RPMS/" + rpm_arch + "/*.rpm") \ - + utils.glob_path("urpmi/builddir/RPMS/noarch/*.rpm"), - "linux/rosa/%s/" % common.channel, - "ROSA") - utils.set_summary("desktop urpmi deploy", rc == 0) + utils.log_h2("desktop urpmi deploy") + ret = aws_s3_upload( + utils.glob_path("urpmi/builddir/RPMS/" + rpm_arch + "/*.rpm") \ + + utils.glob_path("urpmi/builddir/RPMS/noarch/*.rpm"), + "linux/rosa/%s/" % common.channel, + "ROSA" + ) + utils.set_summary("desktop urpmi deploy", ret) - utils.log_h2("desktop rpm-suse deploy") - rc = aws_s3_upload( - utils.glob_path("suse-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ - + utils.glob_path("suse-rpm/builddir/RPMS/noarch/*.rpm"), - "linux/suse/%s/" % common.channel, - "SUSE Linux") - utils.set_summary("desktop rpm-suse deploy", rc == 0) + utils.log_h2("desktop rpm-suse deploy") + ret = aws_s3_upload( + utils.glob_path("suse-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ + + utils.glob_path("suse-rpm/builddir/RPMS/noarch/*.rpm"), + "linux/suse/%s/" % common.channel, + "SUSE Linux" + ) + utils.set_summary("desktop rpm-suse deploy", ret) - if not branding.onlyoffice: - utils.log_h2("desktop deb-astra deploy") - rc = aws_s3_upload( - utils.glob_path("deb-astra/*.deb"), - "linux/astra/", - "Astra Linux Signed") - utils.set_summary("desktop deb-astra deploy", rc == 0) - - else: - utils.set_summary("desktop tar deploy", False) - utils.set_summary("desktop deb deploy", False) - utils.set_summary("desktop rpm deploy", False) - utils.set_summary("desktop rpm-apt deploy", False) - utils.set_summary("desktop urpmi deploy", False) - utils.set_summary("desktop rpm-suse deploy", False) - if not branding.onlyoffice: - utils.set_summary("desktop deb-astra deploy", False) + if not branding.onlyoffice: + utils.log_h2("desktop deb-astra deploy") + ret = aws_s3_upload( + utils.glob_path("deb-astra/*.deb"), + "linux/astra/", + "Astra Linux Signed" + ) + utils.set_summary("desktop deb-astra deploy", ret) + else: + utils.set_summary("desktop tar deploy", False) + utils.set_summary("desktop deb deploy", False) + utils.set_summary("desktop rpm deploy", False) + utils.set_summary("desktop rpm-apt deploy", False) + utils.set_summary("desktop urpmi deploy", False) + utils.set_summary("desktop rpm-suse deploy", False) + if not branding.onlyoffice: + utils.set_summary("desktop deb-astra deploy", False) utils.set_cwd(common.workspace_dir) return diff --git a/scripts/package_mobile.py b/scripts/package_mobile.py index f473334..6f25bc5 100644 --- a/scripts/package_mobile.py +++ b/scripts/package_mobile.py @@ -23,18 +23,23 @@ def make_mobile(): s3_key = "mobile/android/%s/%s" % (common.channel, zip_file) utils.log_h2("mobile build") - rc = utils.sh("zip -r " + zip_file + " ./android* ./js", verbose=True) - utils.set_summary("mobile build", rc == 0) + ret = utils.sh("zip -r " + zip_file + " ./android* ./js", verbose=True) + utils.set_summary("mobile build", ret) - utils.log_h2("mobile deploy") - if rc == 0: - rc = utils.sh( - "aws s3 cp --acl public-read --no-progress " \ - + zip_file + " s3://" + branding.s3_bucket + "/" + s3_key, - verbose=True) - if rc == 0: - utils.add_deploy_data("mobile", "Android", zip_file, s3_key, branding.s3_bucket, branding.s3_region) - utils.set_summary("mobile deploy", rc == 0) + if common.deploy: + utils.log_h2("mobile deploy") + if ret: + ret = utils.sh( + "aws s3 cp --acl public-read --no-progress " \ + + zip_file + " s3://" + branding.s3_bucket + "/" + s3_key, + verbose=True + ) + if ret: + utils.add_deploy_data( + "mobile", "Android", zip_file, s3_key, + branding.s3_bucket, branding.s3_region + ) + utils.set_summary("mobile deploy", ret) utils.set_cwd(common.workspace_dir) return diff --git a/scripts/package_server.py b/scripts/package_server.py index 861a65d..0d15233 100644 --- a/scripts/package_server.py +++ b/scripts/package_server.py @@ -16,23 +16,26 @@ def make(edition): def aws_s3_upload(files, key, edition, ptype=None): key = "server/" + key - rc = 0 for file in files: + args = ["aws"] + if hasattr(branding, "s3_endpoint_url"): + args += ["--endpoint-url=" + branding.s3_endpoint_url] + args += [ + "s3", "cp", "--no-progress", "--acl", "public-read", + file, "s3://" + branding.s3_bucket + "/" + key + ] if common.os_family == "windows": - rc += utils.cmd( - "aws", "s3", "cp", "--acl", "public-read", "--no-progress", - file, "s3://" + branding.s3_bucket + "/" + key, - verbose=True) + ret = utils.cmd(*args, verbose=True) else: - rc += utils.sh( - "aws s3 cp --acl public-read --no-progress " \ - + file + " s3://" + branding.s3_bucket + "/" + key, - verbose=True) - if rc == 0 and ptype is not None: + ret = utils.sh(" ".join(args), verbose=True) + if ret and ptype is not None: full_key = key if full_key.endswith("/"): full_key += utils.get_basename(file) - utils.add_deploy_data("server_" + edition, ptype, file, full_key, branding.s3_bucket, branding.s3_region) - return rc + utils.add_deploy_data( + "server_" + edition, ptype, file, full_key, + branding.s3_bucket, branding.s3_region + ) + return ret def make_windows(edition): if edition == "enterprise": @@ -43,25 +46,23 @@ def make_windows(edition): product_name = "DocumentServer" utils.set_cwd("document-server-package") - utils.log_h2("server " + edition + " clean") - rc = utils.cmd("make", "clean", verbose=True) - utils.set_summary("server " + edition + " clean", rc == 0) - utils.log_h2("server " + edition + " build") + ret = utils.cmd("make", "clean", verbose=True) args = ["-e", "PRODUCT_NAME=" + product_name] if not branding.onlyoffice: args += ["-e", "BRANDING_DIR=../" + common.branding + "/document-server-package"] - rc = utils.cmd("make", "packages", *args, verbose=True) - utils.set_summary("server " + edition + " build", rc == 0) + ret &= utils.cmd("make", "packages", *args, verbose=True) + utils.set_summary("server " + edition + " build", ret) - if rc == 0: + if common.deploy and ret: utils.log_h2("server " + edition + " inno deploy") - rc = aws_s3_upload( + ret = aws_s3_upload( utils.glob_path("exe/*.exe"), "win/inno/%s/" % common.channel, edition, - "Installer") - utils.set_summary("server " + edition + " inno deploy", rc == 0) + "Installer" + ) + utils.set_summary("server " + edition + " inno deploy", ret) utils.set_cwd(common.workspace_dir) return @@ -75,60 +76,61 @@ def make_linux(edition): product_name = "documentserver" utils.set_cwd("document-server-package") - utils.log_h2("server " + edition + " clean") - rc = utils.sh("make clean", verbose=True) - utils.set_summary("server " + edition + " clean", rc == 0) - utils.log_h2("server " + edition + " build") args = ["-e", "PRODUCT_NAME=" + product_name] if common.platform == "linux_aarch64": args += ["-e", "UNAME_M=aarch64"] if not branding.onlyoffice: args += ["-e", "BRANDING_DIR=../" + common.branding + "/document-server-package"] - rc = utils.sh("make packages " + " ".join(args), verbose=True) - utils.set_summary("server " + edition + " build", rc == 0) + ret = utils.sh("make clean && make packages " + " ".join(args), verbose=True) + utils.set_summary("server " + edition + " build", ret) rpm_arch = "x86_64" if common.platform == "linux_aarch64": rpm_arch = "aarch64" - if rc == 0: - utils.log_h2("server " + edition + " tar deploy") - rc = aws_s3_upload( - utils.glob_path("*.tar.gz"), - "linux/generic/%s/" % common.channel, - edition, - "Portable") - utils.set_summary("server " + edition + " tar deploy", rc == 0) + if common.deploy: + utils.log_h2("server " + edition + " deploy") + if ret: + utils.log_h2("server " + edition + " tar deploy") + ret = aws_s3_upload( + utils.glob_path("*.tar.gz"), + "linux/generic/%s/" % common.channel, + edition, + "Portable" + ) + utils.set_summary("server " + edition + " tar deploy", ret) - utils.log_h2("server " + edition + " deb deploy") - rc = aws_s3_upload( - utils.glob_path("deb/*.deb"), - "linux/debian/%s/" % common.channel, - edition, - "Debian") - utils.set_summary("server " + edition + " deb deploy", rc == 0) + utils.log_h2("server " + edition + " deb deploy") + ret = aws_s3_upload( + utils.glob_path("deb/*.deb"), + "linux/debian/%s/" % common.channel, + edition, + "Debian" + ) + utils.set_summary("server " + edition + " deb deploy", ret) - utils.log_h2("server " + edition + " rpm deploy") - rc = aws_s3_upload( - utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), - "linux/rhel/%s/" % common.channel, - edition, - "CentOS") - utils.set_summary("server " + edition + " rpm deploy", rc == 0) + utils.log_h2("server " + edition + " rpm deploy") + ret = aws_s3_upload( + utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), + "linux/rhel/%s/" % common.channel, + edition, + "CentOS" + ) + utils.set_summary("server " + edition + " rpm deploy", ret) - utils.log_h2("server " + edition + " apt-rpm deploy") - rc = aws_s3_upload( - utils.glob_path("apt-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), - "linux/altlinux/%s/" % common.channel, - edition, - "ALT Linux") - utils.set_summary("server " + edition + " apt-rpm deploy", rc == 0) - - else: - utils.set_summary("server " + edition + " tar deploy", False) - utils.set_summary("server " + edition + " deb deploy", False) - utils.set_summary("server " + edition + " rpm deploy", False) - utils.set_summary("server " + edition + " rpm-alt deploy", False) + utils.log_h2("server " + edition + " apt-rpm deploy") + ret = aws_s3_upload( + utils.glob_path("apt-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), + "linux/altlinux/%s/" % common.channel, + edition, + "ALT Linux" + ) + utils.set_summary("server " + edition + " apt-rpm deploy", ret) + else: + utils.set_summary("server " + edition + " tar deploy", False) + utils.set_summary("server " + edition + " deb deploy", False) + utils.set_summary("server " + edition + " rpm deploy", False) + utils.set_summary("server " + edition + " rpm-alt deploy", False) utils.set_cwd(common.workspace_dir) - return \ No newline at end of file + return diff --git a/scripts/package_utils.py b/scripts/package_utils.py index 60d46aa..36d8a09 100644 --- a/scripts/package_utils.py +++ b/scripts/package_utils.py @@ -63,8 +63,8 @@ def get_cwd(): def set_cwd(path, verbose=True): if verbose: - log_h3("change working dir:") - log_h3(" path: " + path) + log("- change working dir:") + log(" path: " + path) os.chdir(path) return @@ -116,8 +116,8 @@ def get_md5(path): def create_dir(path, verbose=True): if verbose: - log_h3("create_dir:") - log_h3(" path:" + path) + log("- create_dir:") + log(" path: " + path) if not is_exist(path): os.makedirs(path) else: @@ -128,21 +128,21 @@ def write_file(path, data, encoding='utf-8', verbose=True): if is_file(path): delete_file(path) if verbose: - log_h3("write_file:") - log_h3(" path: " + path) - log_h3(" encoding: " + encoding) - log_h3(" data: |\n" + data) + log("- write_file:") + log(" path: " + path) + log(" encoding: " + encoding) + log(" data: |\n" + data) with codecs.open(path, 'w', encoding) as file: file.write(data) return def replace_in_file(path, pattern, text_replace, encoding='utf-8', verbose=True): if verbose: - log_h3("replace_in_file:") - log_h3(" path: " + path) - log_h3(" pattern: " + pattern) - log_h3(" replace: " + text_replace) - log_h3(" encoding: " + encoding) + log("- replace_in_file:") + log(" path: " + path) + log(" pattern: " + pattern) + log(" replace: " + text_replace) + log(" encoding: " + encoding) file_data = "" with codecs.open(get_path(path), "r", encoding) as file: file_data = file.read() @@ -154,9 +154,9 @@ def replace_in_file(path, pattern, text_replace, encoding='utf-8', verbose=True) def copy_file(src, dst, verbose=True): if verbose: - log_h3("copy_file:") - log_h3(" src: " + src) - log_h3(" dst: " + dst) + log("- copy_file:") + log(" src: " + src) + log(" dst: " + dst) if is_file(dst): delete_file(dst) if not is_file(src): @@ -166,10 +166,10 @@ def copy_file(src, dst, verbose=True): def copy_files(src, dst, override=True, verbose=True): if verbose: - log_h3("copy_files:") - log_h3(" src: " + src) - log_h3(" dst: " + dst) - log_h3(" override: " + str(override)) + log("- copy_files:") + log(" src: " + src) + log(" dst: " + dst) + log(" override: " + str(override)) for file in glob.glob(src): file_name = os.path.basename(file) if is_file(file): @@ -187,10 +187,10 @@ def copy_files(src, dst, override=True, verbose=True): def copy_dir(src, dst, override=True, verbose=True): if verbose: - log_h3("copy_dir:") - log_h3(" src: " + src) - log_h3(" dst: " + dst) - log_h3(" override: " + str(override)) + log("- copy_dir:") + log(" src: " + src) + log(" dst: " + dst) + log(" override: " + str(override)) if is_dir(dst): delete_dir(dst) try: @@ -201,11 +201,11 @@ def copy_dir(src, dst, override=True, verbose=True): def copy_dir_content(src, dst, filter_include = "", filter_exclude = "", verbose=True): if verbose: - log_h3("copy_dir_content:") - log_h3(" src: " + src) - log_h3(" dst: " + dst) - log_h3(" include: " + filter_include) - log_h3(" exclude: " + filter_exclude) + log("- copy_dir_content:") + log(" src: " + src) + log(" dst: " + dst) + log(" include: " + filter_include) + log(" exclude: " + filter_exclude) src_folder = src if ("/" != src[-1:]): src_folder += "/" @@ -224,8 +224,8 @@ def copy_dir_content(src, dst, filter_include = "", filter_exclude = "", verbose def delete_file(path, verbose=True): if verbose: - log_h3("delete_file:") - log_h3(" path: " + path) + log("- delete_file:") + log(" path: " + path) if not is_file(path): log_err("file not exist") return @@ -233,8 +233,8 @@ def delete_file(path, verbose=True): def delete_dir(path, verbose=True): if verbose: - log_h3("delete_dir:") - log_h3(" path: " + path) + log("- delete_dir:") + log(" path: " + path) if not is_dir(path): log_err("dir not exist") return @@ -243,8 +243,8 @@ def delete_dir(path, verbose=True): def delete_files(src, verbose=True): if verbose: - log_h3("delete_files:") - log_h3(" pattern: " + src) + log("- delete_files:") + log(" pattern: " + src) for path in glob.glob(src): if verbose: log(path) @@ -276,93 +276,96 @@ def add_deploy_data(product, ptype, src, dst, bucket, region): def cmd(*args, **kwargs): if kwargs.get("verbose"): - log_h3("cmd:") - log_h3(" command: " + " ".join(args)) + log("- cmd:") + log(" command: " + " ".join(args)) if kwargs.get("chdir"): - log_h3(" chdir: " + kwargs["chdir"]) + log(" chdir: " + kwargs["chdir"]) if kwargs.get("creates"): - log_h3(" creates: " + kwargs["creates"]) + log(" creates: " + kwargs["creates"]) if kwargs.get("creates") and is_exist(kwargs["creates"]): log_err("creates exist") - return 0 + return False if kwargs.get("chdir") and is_dir(kwargs["chdir"]): oldcwd = get_cwd() set_cwd(kwargs["chdir"]) ret = subprocess.call( [i for i in args], stderr=subprocess.STDOUT, shell=True - ) + ) == 0 if kwargs.get("chdir") and oldcwd: set_cwd(oldcwd) return ret def cmd_output(*args, **kwargs): if kwargs.get("verbose"): - log_h3("cmd_output:") - log_h3(" command: " + " ".join(args)) + log("- cmd_output:") + log(" command: " + " ".join(args)) return subprocess.check_output( [i for i in args], stderr=subprocess.STDOUT, shell=True ).decode("utf-8") def powershell(*args, **kwargs): if kwargs.get("verbose"): - log_h3("powershell:") - log_h3(" command: " + " ".join(args)) + log("- powershell:") + log(" command: " + " ".join(args)) if kwargs.get("chdir"): - log_h3(" chdir: " + kwargs["chdir"]) + log(" chdir: " + kwargs["chdir"]) if kwargs.get("creates"): - log_h3(" creates: " + kwargs["creates"]) + log(" creates: " + kwargs["creates"]) if kwargs.get("creates") and is_exist(kwargs["creates"]): - return 0 + return False args = ["powershell", "-Command"] + [i for i in args] ret = subprocess.call( args, stderr=subprocess.STDOUT, shell=True - ) + ) == 0 return ret def ps1(file, args=[], **kwargs): if kwargs.get("verbose"): log_h2("powershell cmdlet: " + file + " " + " ".join(args)) if kwargs.get("creates") and is_exist(kwargs["creates"]): - return 0 + return True ret = subprocess.call( ["powershell", file] + args, stderr=subprocess.STDOUT, shell=True - ) + ) == 0 return ret def download_file(url, path, md5, verbose=False): if verbose: - log_h3("download_file:") - log_h3(" url: " + path) - log_h3(" path: " + url) - log_h3(" md5: " + md5) + log("- download_file:") + log(" url: " + path) + log(" path: " + url) + log(" md5: " + md5) if is_file(path): if get_md5(path) == md5: log_err("file already exist (match checksum)") - return 0 + return True else: log_err("wrong checksum (%s), delete" % md5) os.remove(path) - ret = powershell("(New-Object System.Net.WebClient).DownloadFile('%s','%s')" % (url, path), verbose=True) + ret = powershell( + "(New-Object System.Net.WebClient).DownloadFile('%s','%s')" % (url, path), + verbose=True + ) md5_new = get_md5(path) if md5 != md5_new: log_err("checksum didn't match (%s != %s)" % (md5, md5_new)) - return 1 + return False return ret def sh(command, **kwargs): if kwargs.get("verbose"): - log_h3("sh:") - log_h3(" command: " + command) + log("- sh:") + log(" command: " + command) if kwargs.get("chdir"): - log_h3(" chdir: " + kwargs["chdir"]) + log(" chdir: " + kwargs["chdir"]) if kwargs.get("creates"): - log_h3(" creates: " + kwargs["creates"]) - return subprocess.call(command, stderr=subprocess.STDOUT, shell=True) + log(" creates: " + kwargs["creates"]) + return subprocess.call(command, stderr=subprocess.STDOUT, shell=True) == 0 def sh_output(command, **kwargs): if kwargs.get("verbose"): - log_h3("sh_output:") - log_h3(" command: " + command) + log("- sh_output:") + log(" command: " + command) return subprocess.check_output( command, stderr=subprocess.STDOUT, shell=True ).decode("utf-8")