Fix core and builder archive deploy (#860)

* Refactoring script parameters

* Add builder 7z deploy

* Refactoring core 7z deploy

* Small fix

make_package.py

@@ -10,15 +10,17 @@ import package_utils as utils
 # parse
 parser = argparse.ArgumentParser(description="Build packages.")
 parser.add_argument("-P", "--platform", dest="platform", type=str,
-                    action="store", help="Defines platform", required=True)
-parser.add_argument("-T", "--targets", dest="targets",   type=str, nargs="+",
-                    action="store", help="Defines targets",  required=True)
-parser.add_argument("-R", "--branding", dest="branding", type=str,
-                    action="store", help="Provides branding path")
+  action="store", help="Defines platform", required=True)
+parser.add_argument("-T", "--targets",  dest="targets",  type=str, nargs="+",
+  action="store", help="Defines targets",  required=True)
 parser.add_argument("-V", "--version",  dest="version",  type=str,
-                    action="store", help="Defines version")
+  action="store", help="Defines version")
 parser.add_argument("-B", "--build",    dest="build",    type=str,
-                    action="store", help="Defines build")
+  action="store", help="Defines build")
+parser.add_argument("-H", "--branch",   dest="branch",   type=str,
+  action="store", help="Defines branch")
+parser.add_argument("-R", "--branding", dest="branding", type=str,
+  action="store", help="Provides branding path")
 args = parser.parse_args()
 
 # vars
@@ -29,8 +31,16 @@ common.targets = args.targets
 common.clean = "clean" in args.targets
 common.sign = "sign" in args.targets
 common.deploy = "deploy" in args.targets
-common.version = args.version if args.version else utils.get_env("BUILD_VERSION", "0.0.0")
-common.build = args.build if args.build else utils.get_env("BUILD_NUMBER", "0")
+if args.version: common.version = args.version
+else:            common.version = utils.get_env("PRODUCT_VERSION", "0.0.0")
+utils.set_env("PRODUCT_VERSION", common.version)
+utils.set_env("BUILD_VERSION", common.version)
+if args.build: common.build = args.build
+else:          common.build = utils.get_env("BUILD_NUMBER", "0")
+utils.set_env("BUILD_NUMBER", common.build)
+if args.branch: common.branch = args.branch
+else:           common.branch = utils.get_env("BRANCH_NAME", "null")
+utils.set_env("BRANCH_NAME", common.branch)
 common.branding = args.branding
 common.timestamp = utils.get_timestamp()
 common.workspace_dir = utils.get_abspath(utils.get_script_dir(__file__) + "/..")

scripts/deploy_core.py

@@ -64,10 +64,5 @@ def make():
 
     # dictionaries
     base.copy_dictionaries(git_dir + "/dictionaries", archive_dir + "/dictionaries", True, False)
-
-    if base.is_file(archive_dir + ".7z"):
-      base.delete_file(archive_dir + ".7z")
-    base.archive_folder(archive_dir + "/*", archive_dir + ".7z")
-
   return
 

scripts/develop/config_server.py

@@ -5,8 +5,9 @@ import base
 import os
 import json
 
-def get_core_url(arch, branch):
-  return "http://repo-doc-onlyoffice-com.s3.amazonaws.com/" + base.host_platform() + "/core/" + branch + "/latest/" + arch + "/core.7z"
+def get_core_url(platform, branch):
+  return "http://repo-doc-onlyoffice-com.s3.amazonaws.com/archive/" \
+    + branch + "/latest/core-" + platform.replace("_", "-") + ".7z"
 
 def make():
   git_dir = base.get_script_dir() + "/../.."
@@ -20,14 +21,21 @@ def make():
 
   arch = "x64"
   arch2 = "_64"
-  if ("windows" == base.host_platform()) and not base.host_platform_is64():
+  if base.is_windows() and not base.host_platform_is64():
     arch = "x86"
     arch2 = "_32"
+  if base.is_os_arm():
+    arch2 = "_arm64"
+  platform = ""
+  if base.is_windows():
+    platform = "win" + arch2
+  else:
+    platform = base.host_platform() + arch2
 
-  url = get_core_url(arch, config.option("branch"))
+  url = get_core_url(platform, config.option("branch"))
   data_url = base.get_file_last_modified_url(url)
   if (data_url == "" and config.option("branch") != "develop"):
-    url = get_core_url(arch, "develop")
+    url = get_core_url(platform, "develop")
     data_url = base.get_file_last_modified_url(url)
   
   old_data_url = base.readFile("./core.7z.data")
@@ -49,12 +57,6 @@ def make():
     base.extract("./core.7z", "./")
     base.writeFile("./core.7z.data", data_url)
 
-    platform = ""
-    if ("windows" == base.host_platform()):
-      platform = "win" + arch2
-    else:
-      platform = base.host_platform() + arch2
-
     base.copy_files("./core/*", "./")
   else:
     print("-----------------------------------------------------------")

scripts/package_builder.py

@@ -7,14 +7,17 @@ import package_branding as branding
 
 def make():
   utils.log_h1("BUILDER")
+  if not (utils.is_windows() or utils.is_macos() or utils.is_linux()):
+    utils.log("Unsupported host OS")
+    return
+  if common.deploy:
+    make_archive()
   if utils.is_windows():
     make_windows()
   elif utils.is_macos():
     make_macos()
   elif utils.is_linux():
     make_linux()
-  else:
-    utils.log("Unsupported host OS")
   return
 
 def s3_upload(files, dst):
@@ -29,6 +32,37 @@ def s3_upload(files, dst):
     ret &= upload
   return ret
 
+def make_archive():
+  utils.set_cwd(utils.get_path(
+    "build_tools/out/" + common.prefix + "/" + branding.company_name.lower()))
+
+  utils.log_h2("builder archive build")
+  utils.delete_file("builder.7z")
+  args = ["7z", "a", "-y", "builder.7z", "./documentbuilder/*"]
+  if utils.is_windows():
+    ret = utils.cmd(*args, verbose=True)
+  else:
+    ret = utils.sh(" ".join(args), verbose=True)
+  utils.set_summary("builder archive build", ret)
+
+  utils.log_h2("builder archive deploy")
+  dest = "builder-" + common.prefix.replace("_","-") + ".7z"
+  dest_latest = "archive/%s/latest/%s" % (common.branch, dest)
+  dest_version = "archive/%s/%s/%s" % (common.branch, common.build, dest)
+  ret = utils.s3_upload(
+    "builder.7z", "s3://" + branding.s3_bucket + "/" + dest_version)
+  utils.set_summary("builder archive deploy", ret)
+  if ret:
+    utils.log("URL: " + branding.s3_base_url + "/" + dest_version)
+    utils.add_deploy_data(dest_version)
+    utils.s3_copy(
+      "s3://" + branding.s3_bucket + "/" + dest_version,
+      "s3://" + branding.s3_bucket + "/" + dest_latest)
+    utils.log("URL: " + branding.s3_base_url + "/" + dest_latest)
+
+  utils.set_cwd(common.workspace_dir)
+  return
+
 def make_windows():
   global inno_file, zip_file, suffix, key_prefix
   utils.set_cwd("document-builder-package")

scripts/package_core.py

@@ -10,47 +10,38 @@ def make():
     utils.log("Unsupported host OS")
     return
   if common.deploy:
-    make_core()
+    make_archive()
   return
 
-def make_core():
-  prefix = common.platformPrefixes[common.platform]
-  company = branding.company_name.lower()
-  repos = {
-    "windows_x64":   { "repo": "windows", "arch": "x64", "version": common.version + "." + common.build },
-    "windows_x86":   { "repo": "windows", "arch": "x86", "version": common.version + "." + common.build },
-    "darwin_x86_64": { "repo": "mac",     "arch": "x64", "version": common.version + "-" + common.build },
-    "darwin_arm64":  { "repo": "mac",     "arch": "arm", "version": common.version + "-" + common.build },
-    "linux_x86_64":  { "repo": "linux",   "arch": "x64", "version": common.version + "-" + common.build },
-  }
-  repo = repos[common.platform]
-  branch = utils.get_env("BRANCH_NAME")
-  core_7z = utils.get_path("build_tools/out/%s/%s/core.7z" % (prefix, company))
-  dest_version = "%s/core/%s/%s/%s" % (repo["repo"], branch, repo["version"], repo["arch"])
-  dest_latest = "%s/core/%s/%s/%s" % (repo["repo"], branch, "latest", repo["arch"])
+def make_archive():
+  utils.set_cwd(utils.get_path(
+    "build_tools/out/" + common.prefix + "/" + branding.company_name.lower()))
 
-  if branch is None:
-    utils.log_err("BRANCH_NAME variable is undefined")
-    utils.set_summary("core deploy", False)
-    return
-  if not utils.is_file(core_7z):
-    utils.log_err("file not exist: " + core_7z)
-    utils.set_summary("core deploy", False)
-    return
+  utils.log_h2("core archive build")
+  utils.delete_file("core.7z")
+  args = ["7z", "a", "-y", "core.7z", "./core/*"]
+  if utils.is_windows():
+    ret = utils.cmd(*args, verbose=True)
+  else:
+    ret = utils.sh(" ".join(args), verbose=True)
+  utils.set_summary("core archive build", ret)
 
-  utils.log_h2("core deploy")
+  utils.log_h2("core archive deploy")
+  dest = "core-" + common.prefix.replace("_","-") + ".7z"
+  dest_latest = "archive/%s/latest/%s" % (common.branch, dest)
+  dest_version = "archive/%s/%s/%s" % (common.branch, common.build, dest)
   ret = utils.s3_upload(
-    core_7z,
-    "s3://" + branding.s3_bucket + "/" + dest_version + "/core.7z")
+    "core.7z", "s3://" + branding.s3_bucket + "/" + dest_version)
+  utils.set_summary("core archive deploy", ret)
   if ret:
-    utils.log("URL: " + branding.s3_base_url + "/" + dest_version + "/core.7z")
-    utils.add_deploy_data(dest_version + "/core.7z")
-    ret = utils.s3_sync(
-      "s3://" + branding.s3_bucket + "/" + dest_version + "/",
-      "s3://" + branding.s3_bucket + "/" + dest_latest + "/",
-      delete=True)
-    utils.log("URL: " + branding.s3_base_url + "/" + dest_latest + "/core.7z")
-  utils.set_summary("core deploy", ret)
+    utils.log("URL: " + branding.s3_base_url + "/" + dest_version)
+    utils.add_deploy_data(dest_version)
+    utils.s3_copy(
+      "s3://" + branding.s3_bucket + "/" + dest_version,
+      "s3://" + branding.s3_bucket + "/" + dest_latest)
+    utils.log("URL: " + branding.s3_base_url + "/" + dest_latest)
+
+  utils.set_cwd(common.workspace_dir)
   return
 
 def deploy_closuremaps_sdkjs(license):

scripts/package_utils.py

@@ -385,15 +385,13 @@ def s3_upload(src, dst, **kwargs):
     ret = sh(" ".join(args), verbose=True)
   return ret
 
-def s3_sync(src, dst, **kwargs):
+def s3_copy(src, dst, **kwargs):
   args = ["aws"]
   if kwargs.get("endpoint_url"):
     args += ["--endpoint-url", kwargs["endpoint_url"]]
-  args += ["s3", "sync", "--no-progress"]
+  args += ["s3", "cp", "--no-progress"]
   if kwargs.get("acl"):
     args += ["--acl", kwargs["acl"]]
-  if kwargs.get("delete") and kwargs["delete"]:
-    args += ["--delete"]
   args += [src, dst]
   if is_windows():
     ret = cmd(*args, verbose=True)