From 1431d3a541d9222bae0df5938a2e2cb084e97634 Mon Sep 17 00:00:00 2001 From: Semyon Bezrukov Date: Thu, 13 Apr 2023 17:06:56 +0300 Subject: [PATCH] Package deploy refactoring (#649) * Deploy closure maps (#635) * Package deploy refactoring --- make_package.py | 4 +++ scripts/package_builder.py | 22 +++++------------ scripts/package_core.py | 45 +++++++++++++++++++++++++++++++++- scripts/package_desktop.py | 50 ++++++++++++-------------------------- scripts/package_mobile.py | 7 ++---- scripts/package_server.py | 25 +++++-------------- scripts/package_utils.py | 4 +-- 7 files changed, 79 insertions(+), 78 deletions(-) diff --git a/make_package.py b/make_package.py index 71a32ce..0062aa3 100755 --- a/make_package.py +++ b/make_package.py @@ -64,6 +64,10 @@ utils.set_cwd(common.workspace_dir, verbose=True) utils.delete_file("deploy.json") if "core" in common.targets: package_core.make() +if "closure-maps-os" in common.targets: + package_core.deploy_closure_maps("opensource") +if "closure-maps-com" in common.targets: + package_core.deploy_closure_maps("commercial") if "desktop" in common.targets: package_desktop.make() if "builder" in common.targets: diff --git a/scripts/package_builder.py b/scripts/package_builder.py index 0ac03bc..999a416 100644 --- a/scripts/package_builder.py +++ b/scripts/package_builder.py @@ -36,10 +36,7 @@ def aws_s3_upload(files, key, ptype=None): if upload and ptype is not None: full_key = key if full_key.endswith("/"): full_key += utils.get_basename(file) - utils.add_deploy_data( - "builder", ptype, file, full_key, - branding.s3_bucket, branding.s3_region - ) + utils.add_deploy_data("builder", ptype, file, full_key) return ret def make_windows(): @@ -83,9 +80,7 @@ def make_zip(): if common.deploy and ret: utils.log_h2("builder zip deploy") - ret = aws_s3_upload( - ["build\\" + zip_file], "win/generic/%s/" % common.channel, "Portable" - ) + ret = aws_s3_upload(["build\\" + zip_file], "win/generic/", "Portable") utils.set_summary("builder zip deploy", ret) return @@ -110,9 +105,7 @@ def make_inno(): if common.deploy and ret: utils.log_h2("builder inno deploy") - ret = aws_s3_upload( - ["build\\" + inno_file], "win/inno/%s/" % common.channel, "Installer" - ) + ret = aws_s3_upload(["build\\" + inno_file], "win/inno/", "Installer") utils.set_summary("builder inno deploy", ret) return @@ -138,24 +131,21 @@ def make_linux(): utils.log_h2("builder tar deploy") ret = aws_s3_upload( utils.glob_path("tar/*.tar.gz"), - "linux/generic/%s/" % common.channel, - "Portable" + "linux/generic/", "Portable" ) utils.set_summary("builder tar deploy", ret) if "deb" in branding.builder_make_targets: utils.log_h2("builder deb deploy") ret = aws_s3_upload( utils.glob_path("deb/*.deb"), - "linux/debian/%s/" % common.channel, - "Debian" + "linux/debian/", "Debian" ) utils.set_summary("builder deb deploy", ret) if "rpm" in branding.builder_make_targets: utils.log_h2("builder rpm deploy") ret = aws_s3_upload( utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), - "linux/rhel/%s/" % common.channel, - "CentOS" + "linux/rhel/", "CentOS" ) utils.set_summary("builder rpm deploy", ret) else: diff --git a/scripts/package_core.py b/scripts/package_core.py index eb25899..3e5372b 100644 --- a/scripts/package_core.py +++ b/scripts/package_core.py @@ -46,7 +46,7 @@ def make_core(): else: ret = utils.sh(" ".join(args), verbose=True) if ret: - utils.add_deploy_data("core", "Archive", core_7z, dest_version + "core.7z", branding.s3_bucket, branding.s3_region) + utils.add_deploy_data("core", "Archive", core_7z, dest_version + "core.7z") args = ["aws", "s3", "sync", "--delete", "--acl", "public-read", "--no-progress", "s3://" + branding.s3_bucket + "/" + dest_version, @@ -57,3 +57,46 @@ def make_core(): ret &= utils.sh(" ".join(args), verbose=True) utils.set_summary("core deploy", ret) return + +def deploy_closure_maps(license): + if not common.deploy: return + utils.log_h1("CLOSURE MAPS") + utils.set_cwd(utils.get_path("sdkjs/build")) + + branch = utils.get_env("BRANCH_NAME") + maps = utils.glob_path("*.js.map") + + if branch is None: + utils.log_err("BRANCH_NAME variable is undefined") + utils.set_summary("closure maps " + license + " deploy", False) + return + if not maps: + utils.log_err("files do not exist") + utils.set_summary("closure maps " + license + " deploy", False) + return + + utils.log_h2("closure maps " + license + " deploy") + dest = "closure-maps/%s/%s/%s" % (common.version, common.build, license) + ret = True + for file in maps: + args = ["aws"] + if hasattr(branding, "s3_endpoint_url"): + args += ["--endpoint-url=" + branding.s3_endpoint_url] + args += [ + "s3", "cp", "--no-progress", file, + "s3://" + branding.s3_bucket + "/" + dest + "/" + ] + if common.os_family == "windows": + upload = utils.cmd(*args, verbose=True) + else: + upload = utils.sh(" ".join(args), verbose=True) + ret &= upload + if upload: + utils.add_deploy_data( + "core", "Closure maps " + license, file, dest + "/" + file, + branding.s3_bucket, branding.s3_region + ) + utils.set_summary("closure maps " + license + " deploy", ret) + + utils.set_cwd(common.workspace_dir) + return diff --git a/scripts/package_desktop.py b/scripts/package_desktop.py index 5a2de4e..7aedfe5 100644 --- a/scripts/package_desktop.py +++ b/scripts/package_desktop.py @@ -39,10 +39,7 @@ def aws_s3_upload(files, key, ptype=None): if upload and ptype is not None: full_key = key if full_key.endswith("/"): full_key += utils.get_basename(file) - utils.add_deploy_data( - "desktop", ptype, file, full_key, - branding.s3_bucket, branding.s3_region - ) + utils.add_deploy_data("desktop", ptype, file, full_key) return ret # @@ -56,8 +53,8 @@ def make_windows(): utils.set_cwd("desktop-apps\\win-linux\\package\\windows") prefix = common.platforms[common.platform]["prefix"] - company = branding.company_name.lower() - product = branding.desktop_product_name.replace(" ","").lower() + company = branding.company_name + product = branding.desktop_product_name.replace(" ","") package_name = branding.desktop_package_name package_version = common.version + "." + common.build source_dir = "..\\..\\..\\..\\build_tools\\out\\%s\\%s\\%s" % (prefix, company, product) @@ -135,11 +132,7 @@ def make_zip(): if common.deploy and ret: utils.log_h2("desktop zip deploy") - ret = aws_s3_upload( - [zip_file], - "win/generic/%s/" % common.channel, - "Portable" - ) + ret = aws_s3_upload([zip_file], "win/generic/", "Portable") utils.set_summary("desktop zip deploy", ret) return @@ -403,11 +396,7 @@ def make_advinst(): if common.deploy and ret: utils.log_h2("desktop advinst deploy") - ret = aws_s3_upload( - [advinst_file], - "win/advinst/%s/" % common.channel, - "Installer" - ) + ret = aws_s3_upload([advinst_file], "win/advinst/", "Installer") utils.set_summary("desktop advinst deploy", ret) return @@ -483,7 +472,7 @@ def make_dmg(): utils.log_h2("desktop dmg deploy") ret = aws_s3_upload( utils.glob_path("build/*.dmg"), - "mac/%s/%s/%s/" % (suffix, common.version, common.build), + "mac/%s/%s/%s/" % (common.version, common.build, suffix), "Disk Image" ) utils.set_summary("desktop dmg deploy", ret) @@ -491,7 +480,7 @@ def make_dmg(): utils.log_h2("desktop zip deploy") ret = aws_s3_upload( ["build/%s-%s.zip" % (scheme, app_version)], - "mac/%s/%s/%s/" % (suffix, common.version, common.build), + "mac/%s/%s/%s/" % (common.version, common.build, suffix), "Archive" ) utils.set_summary("desktop zip deploy", ret) @@ -556,7 +545,7 @@ def make_sparkle_updates(): utils.glob_path("build/update/*.delta") \ + utils.glob_path("build/update/*.xml") \ + utils.glob_path("build/update/*.html"), - "mac/%s/%s/%s/" % (suffix, common.version, common.build), + "mac/%s/%s/%s/" % (common.version, common.build, suffix), "Sparkle" ) utils.set_summary("desktop sparkle files deploy", ret) @@ -564,7 +553,7 @@ def make_sparkle_updates(): utils.log_h2("desktop checksums deploy") ret = aws_s3_upload( utils.glob_path("build/update/*.txt"), - "mac/%s/%s/%s/" % (suffix, common.version, common.build), + "mac/%s/%s/%s/" % (common.version, common.build, suffix), "Checksums" ) utils.set_summary("desktop checksums deploy", ret) @@ -596,24 +585,21 @@ def make_linux(): if "tar" in branding.desktop_make_targets: ret = aws_s3_upload( utils.glob_path("tar/*.tar.gz") + utils.glob_path("tar/*.tar.xz"), - "linux/generic/%s/" % common.channel, - "Portable" + "linux/generic/", "Portable" ) utils.set_summary("desktop tar deploy", ret) if "deb" in branding.desktop_make_targets: utils.log_h2("desktop deb deploy") ret = aws_s3_upload( utils.glob_path("deb/*.deb"), - "linux/debian/%s/" % common.channel, - "Debian" + "linux/debian/", "Debian" ) utils.set_summary("desktop deb deploy", ret) if "deb-astra" in branding.desktop_make_targets: utils.log_h2("desktop deb-astra deploy") ret = aws_s3_upload( utils.glob_path("deb-astra/*.deb"), - "linux/astra/", - "Astra Linux Signed" + "linux/astra/", "Astra Linux Special Edition" ) utils.set_summary("desktop deb-astra deploy", ret) if "rpm" in branding.desktop_make_targets: @@ -621,8 +607,7 @@ def make_linux(): ret = aws_s3_upload( utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ + utils.glob_path("rpm/builddir/RPMS/noarch/*.rpm"), - "linux/rhel/%s/" % common.channel, - "CentOS" + "linux/rhel/", "CentOS" ) utils.set_summary("desktop rpm deploy", ret) if "suse-rpm" in branding.desktop_make_targets: @@ -630,8 +615,7 @@ def make_linux(): ret = aws_s3_upload( utils.glob_path("suse-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ + utils.glob_path("suse-rpm/builddir/RPMS/noarch/*.rpm"), - "linux/suse/%s/" % common.channel, - "SUSE Linux" + "linux/suse/", "SUSE Linux" ) utils.set_summary("desktop suse-rpm deploy", ret) if "apt-rpm" in branding.desktop_make_targets: @@ -639,8 +623,7 @@ def make_linux(): ret = aws_s3_upload( utils.glob_path("apt-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm") \ + utils.glob_path("apt-rpm/builddir/RPMS/noarch/*.rpm"), - "linux/altlinux/%s/" % common.channel, - "ALT Linux" + "linux/altlinux/", "ALT Linux" ) utils.set_summary("desktop apt-rpm deploy", ret) if "urpmi" in branding.desktop_make_targets: @@ -648,8 +631,7 @@ def make_linux(): ret = aws_s3_upload( utils.glob_path("urpmi/builddir/RPMS/" + rpm_arch + "/*.rpm") \ + utils.glob_path("urpmi/builddir/RPMS/noarch/*.rpm"), - "linux/rosa/%s/" % common.channel, - "ROSA" + "linux/rosa/", "ROSA" ) utils.set_summary("desktop urpmi deploy", ret) else: diff --git a/scripts/package_mobile.py b/scripts/package_mobile.py index 6f25bc5..54e5e07 100644 --- a/scripts/package_mobile.py +++ b/scripts/package_mobile.py @@ -20,7 +20,7 @@ def make_mobile(): utils.sh("rm -rfv *.zip", verbose=True) zip_file = "build-" + common.version + "-" + common.build + ".zip" - s3_key = "mobile/android/%s/%s" % (common.channel, zip_file) + s3_key = "mobile/android/" + zip_file utils.log_h2("mobile build") ret = utils.sh("zip -r " + zip_file + " ./android* ./js", verbose=True) @@ -35,10 +35,7 @@ def make_mobile(): verbose=True ) if ret: - utils.add_deploy_data( - "mobile", "Android", zip_file, s3_key, - branding.s3_bucket, branding.s3_region - ) + utils.add_deploy_data("mobile", "Android", zip_file, s3_key) utils.set_summary("mobile deploy", ret) utils.set_cwd(common.workspace_dir) diff --git a/scripts/package_server.py b/scripts/package_server.py index 6c1d59e..6ea590c 100644 --- a/scripts/package_server.py +++ b/scripts/package_server.py @@ -35,10 +35,7 @@ def aws_s3_upload(files, key, edition, ptype=None): if upload and ptype is not None: full_key = key if full_key.endswith("/"): full_key += utils.get_basename(file) - utils.add_deploy_data( - "server_" + edition, ptype, file, full_key, - branding.s3_bucket, branding.s3_region - ) + utils.add_deploy_data("server_" + edition, ptype, file, full_key) return ret def make_windows(edition): @@ -62,9 +59,7 @@ def make_windows(edition): utils.log_h2("server " + edition + " inno deploy") ret = aws_s3_upload( utils.glob_path("exe/*.exe"), - "win/inno/%s/" % common.channel, - edition, - "Installer" + "win/inno/", edition, "Installer" ) utils.set_summary("server " + edition + " inno deploy", ret) @@ -99,36 +94,28 @@ def make_linux(edition): utils.log_h2("server " + edition + " deb deploy") ret = aws_s3_upload( utils.glob_path("deb/*.deb"), - "linux/debian/%s/" % common.channel, - edition, - "Debian" + "linux/debian/", edition, "Debian" ) utils.set_summary("server " + edition + " deb deploy", ret) if "rpm" in branding.server_make_targets: utils.log_h2("server " + edition + " rpm deploy") ret = aws_s3_upload( utils.glob_path("rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), - "linux/rhel/%s/" % common.channel, - edition, - "CentOS" + "linux/rhel/", edition, "CentOS" ) utils.set_summary("server " + edition + " rpm deploy", ret) if "apt-rpm" in branding.server_make_targets: utils.log_h2("server " + edition + " apt-rpm deploy") ret = aws_s3_upload( utils.glob_path("apt-rpm/builddir/RPMS/" + rpm_arch + "/*.rpm"), - "linux/altlinux/%s/" % common.channel, - edition, - "ALT Linux" + "linux/altlinux/", edition, "ALT Linux" ) utils.set_summary("server " + edition + " apt-rpm deploy", ret) if "tar" in branding.server_make_targets: utils.log_h2("server " + edition + " snap deploy") ret = aws_s3_upload( utils.glob_path("*.tar.gz"), - "linux/generic/%s/" % common.channel, - edition, - "Snap" + "linux/generic/", edition, "Snap" ) utils.set_summary("server " + edition + " snap deploy", ret) else: diff --git a/scripts/package_utils.py b/scripts/package_utils.py index d52a654..f3fa4b8 100644 --- a/scripts/package_utils.py +++ b/scripts/package_utils.py @@ -258,15 +258,13 @@ def set_summary(target, status): common.summary.append({target: status}) return -def add_deploy_data(product, ptype, src, dst, bucket, region): +def add_deploy_data(product, ptype, src, dst): common.deploy_data.append({ "platform": common.platforms[common.platform]["title"], "product": product, "type": ptype, # "local": get_path(src), "size": get_file_size(get_path(src)), - "bucket": bucket, - "region": region, "key": dst }) file = open(get_path(common.workspace_dir + "/deploy.json"), 'w')