Fix bug with MySQL

delete generating missing files with json

configure.py

@@ -27,7 +27,7 @@ parser.add_option("--db-pass", action="store", type="string", dest="db-pass", de
 parser.add_option("--compiler", action="store", type="string", dest="compiler", default="", help="defines compiler name. It is not recommended to use it as it's defined automatically (msvc2015, msvc2015_64, gcc, gcc_64, clang, clang_64, etc)")
 parser.add_option("--no-apps", action="store", type="string", dest="no-apps", default="0", help="disables building desktop apps that use qt")
 parser.add_option("--themesparams", action="store", type="string", dest="themesparams", default="", help="provides settings for generating presentation themes thumbnails")
-parser.add_option("--git-protocol", action="store", type="string", dest="git-protocol", default="https", help="can be used only if update is set to true - 'https', 'ssh'")
+parser.add_option("--git-protocol", action="store", type="string", dest="git-protocol", default="auto", help="can be used only if update is set to true - 'https', 'ssh'")
 parser.add_option("--branding", action="store", type="string", dest="branding", default="", help="provides branding path")
 parser.add_option("--branding-name", action="store", type="string", dest="branding-name", default="", help="provides branding name")
 parser.add_option("--branding-url", action="store", type="string", dest="branding-url", default="", help="provides branding url")

scripts/base.py

@@ -39,6 +39,9 @@ def is_os_arm():
     return False
   return True
 
+def get_platform():
+  return platform.machine().lower()
+
 def is_python_64bit():
   return (struct.calcsize("P") == 8)
 
@@ -497,12 +500,37 @@ def set_cwd(dir):
   return
 
 # git ---------------------------------------------------
+def git_get_origin():
+  cur_dir = os.getcwd()
+  os.chdir(get_script_dir() + "/../")
+  ret = run_command("git config --get remote.origin.url")["stdout"]
+  os.chdir(cur_dir)
+  return ret
+
+def git_is_ssh():
+  git_protocol = config.option("git-protocol")
+  if (git_protocol == "https"):
+    return False
+  if (git_protocol == "ssh"):
+    return True
+  origin = git_get_origin()
+  if (git_protocol == "auto") and (origin.find(":ONLYOFFICE/") != -1):
+    return True
+  return False
+
+def get_ssh_base_url():
+  cur_origin = git_get_origin()
+  ind = cur_origin.find(":ONLYOFFICE/")
+  if (ind == -1):
+    return "git@github.com:ONLYOFFICE/"
+  return cur_origin[:ind+12]
+
 def git_update(repo, is_no_errors=False, is_current_dir=False, git_owner=""):
   print("[git] update: " + repo)
   owner = git_owner if git_owner else "ONLYOFFICE"
   url = "https://github.com/" + owner + "/" + repo + ".git"
-  if config.option("git-protocol") == "ssh":
-    url = "git@github.com:ONLYOFFICE/" + repo + ".git"
+  if git_is_ssh():
+    url = get_ssh_base_url() + repo + ".git"
   folder = get_script_dir() + "/../../" + repo
   if is_current_dir:
     folder = repo
@@ -571,8 +599,8 @@ def get_branding_repositories(checker):
 def create_pull_request(branches_to, repo, is_no_errors=False, is_current_dir=False):
   print("[git] create pull request: " + repo)
   url = "https://github.com/ONLYOFFICE/" + repo + ".git"
-  if config.option("git-protocol") == "ssh":
-    url = "git@github.com:ONLYOFFICE/" + repo + ".git"
+  if git_is_ssh():
+    url = get_ssh_base_url() + repo + ".git"
   folder = get_script_dir() + "/../../" + repo
   if is_current_dir:
     folder = repo
@@ -1333,7 +1361,7 @@ def copy_marketplace_plugin(dst_dir, is_name_as_guid=False, is_desktop_local=Fal
   git_dir = __file__script__path__ + "/../.."
   if False:
     # old version
-    base.copy_sdkjs_plugin(git_dir + "/desktop-sdk/ChromiumBasedEditors/plugins", dst_dir, "manager", is_name_as_guid, is_desktop_local)
+    copy_sdkjs_plugin(git_dir + "/desktop-sdk/ChromiumBasedEditors/plugins", dst_dir, "manager", is_name_as_guid, is_desktop_local)
     return
   src_dir_path = git_dir + "/onlyoffice.github.io/store/plugin"
   name = "marketplace"
@@ -1760,3 +1788,42 @@ def apply_patch(file, patch):
   #file_content_new = "\n#if 0" + file_content_old + "#else" + file_content_new + "#endif\n"
   replaceInFile(file, file_content_old, file_content_new)
   return
+
+def get_autobuild_version(product, platform="", branch="", build=""):
+  download_platform = platform
+  if ("" == download_platform):
+    osType = get_platform()
+    isArm = True if (-1 != osType.find("arm")) else False
+    is64 = True if (osType.endswith("64")) else False
+    
+    if ("windows" == host_platform()):
+      download_platform = "win-"
+    elif ("linux" == host_platform()):
+      download_platform = "linux-"
+    else:
+      download_platform = "mac-"
+
+    download_platform += ("arm" if isArm else "")
+    download_platform += ("64" if is64 else "32")
+  else:
+    download_platform = download_platform.replace("_", "-")
+
+  download_build = build
+  if ("" == download_build):
+    download_build = "latest"
+
+  download_branch = branch
+  if ("" == download_branch):
+    download_branch = "develop"
+
+  download_addon = download_branch + "/" + download_build + "/" + product + "-" + download_platform + ".7z"
+  return "http://repo-doc-onlyoffice-com.s3.amazonaws.com/archive/" + download_addon
+
+def create_x2t_js_cache(dir, product):
+  if is_file(dir + "/libdoctrenderer.dylib") and (os.path.getsize(dir + "/libdoctrenderer.dylib") < 5*1024*1024):
+    return
+
+  if (product in ["builder", "server"]):
+    cmd_in_dir(dir, "./x2t", ["-create-js-cache"], True)
+  cmd_in_dir(dir, "./x2t", ["-create-js-snapshots"], True)
+  return

scripts/deploy_builder.py

@@ -123,6 +123,8 @@ def make():
     if (0 == platform.find("mac")):
       base.mac_correct_rpath_x2t(root_dir)
       base.mac_correct_rpath_docbuilder(root_dir)
+
+    base.create_x2t_js_cache(root_dir, "builder")
   
   return
 

scripts/deploy_core.py

@@ -64,6 +64,11 @@ def make():
     base.copy_exe(core_build_dir + "/bin/" + platform_postfix, archive_dir, "metafiletester")
     base.copy_exe(core_build_dir + "/bin/" + platform_postfix, archive_dir, "dictionariestester")
 
+    # js cache
+    base.generate_doctrenderer_config(archive_dir + "/DoctRenderer.config", "./", "builder", "", "./dictionaries")
+    base.create_x2t_js_cache(archive_dir, "core")
+    base.delete_file(archive_dir + "/DoctRenderer.config")
+
     # dictionaries
     base.copy_dictionaries(git_dir + "/dictionaries", archive_dir + "/dictionaries", True, False)
   return

scripts/deploy_desktop.py

@@ -260,6 +260,8 @@ def make():
     if isUseJSC:
       base.delete_file(root_dir + "/converter/icudtl.dat")
 
+    base.create_x2t_js_cache(root_dir + "/converter", "desktop")
+
     if (0 == platform.find("win")):
       base.delete_file(root_dir + "/cef_sandbox.lib")
       base.delete_file(root_dir + "/libcef.lib")

scripts/deploy_server.py

@@ -114,10 +114,13 @@ def make():
     js_dir = root_dir
     base.copy_dir(base_dir + "/js/" + branding + "/builder/sdkjs", js_dir + "/sdkjs")
     base.copy_dir(base_dir + "/js/" + branding + "/builder/web-apps", js_dir + "/web-apps")
+    base.copy_file(js_dir + "/web-apps/apps/api/documents/api.js", js_dir + "/web-apps/apps/api/documents/api.js.tpl")
     for file in glob.glob(js_dir + "/web-apps/apps/*/*/*.js.map") \
               + glob.glob(js_dir + "/web-apps/apps/*/mobile/dist/js/*.js.map"):
       base.delete_file(file)
 
+    base.create_x2t_js_cache(converter_dir, "server")
+
     # add embed worker code
     base.cmd_in_dir(git_dir + "/sdkjs/common/embed", "python", ["make.py", js_dir + "/web-apps/apps/api/documents/api.js"])
     

scripts/develop/config_server.py

@@ -5,10 +5,6 @@ import base
 import os
 import json
 
-def get_core_url(platform, branch):
-  return "http://repo-doc-onlyoffice-com.s3.amazonaws.com/archive/" \
-    + branch + "/latest/core-" + platform.replace("_", "-") + ".7z"
-
 def make():
   git_dir = base.get_script_dir() + "/../.."
   old_cur = os.getcwd()
@@ -19,23 +15,10 @@ def make():
 
   os.chdir(work_dir)
 
-  arch = "x64"
-  arch2 = "_64"
-  if base.is_windows() and not base.host_platform_is64():
-    arch = "x86"
-    arch2 = "_32"
-  if base.is_os_arm():
-    arch2 = "_arm64"
-  platform = ""
-  if base.is_windows():
-    platform = "win" + arch2
-  else:
-    platform = base.host_platform() + arch2
-
-  url = get_core_url(platform, config.option("branch"))
+  url = base.get_autobuild_version("core", "", config.option("branch"))
   data_url = base.get_file_last_modified_url(url)
   if (data_url == "" and config.option("branch") != "develop"):
-    url = get_core_url(platform, "develop")
+    url = base.get_autobuild_version("core", "", "develop")
     data_url = base.get_file_last_modified_url(url)
   
   old_data_url = base.readFile("./core.7z.data")

scripts/develop/dependence.py

@@ -904,7 +904,16 @@ def install_gruntcli():
 
 def install_mysqlserver():
   if (host_platform == 'windows'):
-    return os.system('"' + os.environ['ProgramFiles(x86)'] + '\\MySQL\\MySQL Installer for Windows\\MySQLInstallerConsole" community install server;' + install_params['MySQLServer']['version'] + ';x64:*:type=config;openfirewall=true;generallog=true;binlog=true;serverid=' + config.option("db-port") + 'enable_tcpip=true;port=' + config.option("db-port") + ';rootpasswd=' + config.option("db-pass") + ' -silent')
+    program_files = os.environ["ProgramFiles(x86)"]
+    mysql_installer_path = f"\"{program_files}\MySQL\MySQL Installer for Windows\MySQLInstallerConsole.exe\""
+    mysql_installer_params = " community install server;"
+    mysql_installer_params += "8.0.21" + ";"
+    mysql_installer_params += "x64:*:type=config;openfirewall=true;generallog=true;binlog=true;serverid="
+    mysql_installer_params += "3306" + ';'
+    mysql_installer_params += "enable_tcpip=true;port=" + "3306" + ";"
+    mysql_installer_params += "rootpasswd=" + "onlyoffice"
+    mysql_installer_params += " -silent"
+    base.cmd2(mysql_installer_path, [mysql_installer_params])
   elif (host_platform == 'linux'):
     os.system('sudo kill ' + base.run_command('sudo fuser -vn tcp ' + config.option("db-port"))['stdout'])
     code = os.system('sudo ufw enable && sudo ufw allow 22 && sudo ufw allow 3306')

scripts/sdkjs_common/jsdoc/README.md

@@ -3,14 +3,20 @@
 
 This guide explains how to generate documentation for Onlyoffice Builder/Plugins API using the provided Python scripts: `generate_docs_json.py`, `generate_docs_plugins_json.py`, `generate_docs_md.py`. These scripts are used to create JSON and Markdown documentation for the `apiBuilder.js` files from the word, cell, and slide editors.
 
-## Prerequisites
+## Requirements
 
-1. **Node.js and npm**: Ensure you have Node.js and npm installed on your machine. You can download them from [Node.js official website](https://nodejs.org/).
+```bash
+Node.js v20 and above
+Python v3.10 and above
+```
 
-2. **jsdoc**: The scripts use `jsdoc` to generate documentation. Install it using npm:
-   ```bash
-   npm install
-   ```
+## Installation
+
+```bash
+git clone https://github.com/ONLYOFFICE/build_tools.git
+cd build_tools/scripts/sdkjs_common/jsdoc
+npm install
+```
 
 ## Scripts Overview
 
@@ -24,7 +30,7 @@ This script generates JSON documentation based on the `apiBuilder.js` files.
   ```
 
 - **Parameters**:
-  - `output_path` (optional): The directory where the JSON documentation will be saved. If not specified, the default path is `Onlyoffice/document-builder-declarations/document-builder`.
+  - `output_path` (optional): The directory where the JSON documentation will be saved. If not specified, the default path is `../../../../office-js-api-declarations/office-js-api`.
 
 ### `generate_docs_plugins_json.py`
 
@@ -36,7 +42,7 @@ This script generates JSON documentation based on the `api_plugins.js` files.
   ```
 
 - **Parameters**:
-  - `output_path` (optional): The directory where the JSON documentation will be saved. If not specified, the default path is `Onlyoffice/document-builder-declarations/document-builder-plugin`.
+  - `output_path` (optional): The directory where the JSON documentation will be saved. If not specified, the default path is `../../../../office-js-api-declarations/office-js-api-plugins`.
 
 ### `generate_docs_md.py`
 
@@ -48,7 +54,7 @@ This script generates Markdown documentation from the `apiBuilder.js` files.
   ```
 
 - **Parameters**:
-  - `output_path` (optional): The directory where the Markdown documentation will be saved. If not specified, the default path is `Onlyoffice/office-js-api`.
+  - `output_path` (optional): The directory where the Markdown documentation will be saved. If not specified, the default path is `../../../../office-js-api/`.
 
 ## Example
 

scripts/sdkjs_common/jsdoc/generate_docs_json.py

@@ -3,6 +3,7 @@ import subprocess
 import json
 import argparse
 import re
+import platform
 
 root = '../../../..'
 
@@ -22,20 +23,17 @@ editors_maps = {
 }
 
 def generate(output_dir, md=False):
-    missing_examples_file = f'{output_dir}/missing_examples.txt'
-
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
 
-    # Recreate missing_examples.txt file
-    with open(missing_examples_file, 'w', encoding='utf-8') as f:
-        f.write('')
-
     # Generate JSON documentation
     for config in configs:
         editor_name = config.split('/')[-1].replace('.json', '')
         output_file = os.path.join(output_dir, editor_name + ".json")
-        command = f"set EDITOR={editors_maps[editor_name]} && npx jsdoc -c {config} -X > {output_file}"
+        command_set_env = "export"
+        if (platform.system().lower() == "windows"):
+            command_set_env = "set"
+        command = f"{command_set_env} EDITOR={editors_maps[editor_name]} && npx jsdoc -c {config} -X > {output_file}"
         print(f"Generating {editor_name}.json: {command}")
         subprocess.run(command, shell=True)
 
@@ -80,11 +78,6 @@ def generate(output_dir, md=False):
                             if "forms" == document_type:
                                 document_type = "pdf"
                             doclet['description'] = doclet['description'] + f'\n\n## Try it\n\n ```js document-builder={{"documentType": "{document_type}"}}\n{code_content}\n```'
-                        
-                    else:
-                        # Record missing examples in missing_examples.txt
-                        with open(missing_examples_file, 'a', encoding='utf-8') as missing_file:
-                            missing_file.write(f"{file_path}\n")
         
         # Write the modified JSON file back
         with open(output_file, 'w', encoding='utf-8') as f:
@@ -103,25 +96,6 @@ def remove_js_comments(text):
     # Remove multi-line comments, leaving text after /*
     text = re.sub(r'/\*\s*|\s*\*/', '', text, flags=re.DOTALL)
     return text.strip()
-
-def get_current_branch(path):
-    try:
-        # Navigate to the specified directory and get the current branch name
-        result = subprocess.run(
-            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
-            cwd=path,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True
-        )
-        if result.returncode == 0:
-            return result.stdout.strip()
-        else:
-            print(f"Error: {result.stderr}")
-            return None
-    except Exception as e:
-        print(f"Exception: {e}")
-        return None
     
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Generate documentation")
@@ -130,15 +104,7 @@ if __name__ == "__main__":
         type=str, 
         help="Destination directory for the generated documentation",
         nargs='?',  # Indicates the argument is optional
-        default=f"{root}/document-builder-declarations/document-builder"  # Default value
+        default=f"{root}/office-js-api-declarations/office-js-api"
     )
     args = parser.parse_args()
-    
-    branch_name = get_current_branch(f"{root}/sdkjs")
-    if branch_name:
-        index_last_name = branch_name.rfind("/")
-        if -1 != index_last_name:
-            branch_name = branch_name[index_last_name + 1:]
-        args.destination = f"{args.destination}/{branch_name}"
-    
     generate(args.destination)

scripts/sdkjs_common/jsdoc/generate_docs_md.py

@@ -13,6 +13,8 @@ editors = [
     "forms"
 ]
 
+missing_examples = []
+
 def load_json(file_path):
     with open(file_path, 'r', encoding='utf-8') as f:
         return json.load(f)
@@ -56,7 +58,7 @@ def remove_line_breaks(string):
     return re.sub(r'[\r\n]', ' ', string)
 
 def generate_data_types_markdown(types, enumerations, classes, root='../../'):
-    param_types_md = ' |'.join(types)
+    param_types_md = ' | '.join(types)
 
     for enum in enumerations:
         if enum['name'] in types:
@@ -72,7 +74,7 @@ def generate_data_types_markdown(types, enumerations, classes, root='../../'):
             return f"<[{element}]({root}Enumeration/{base_type}.md)>"
         elif base_type in classes:
             return f"<[{element}]({root}{base_type}/{base_type}.md)>"
-        return f"<{element}>"
+        return f"&lt;{element}&gt;"
     
     return re.sub(r'<([^<>]+)>', replace_with_links, param_types_md)
 
@@ -196,10 +198,11 @@ def generate_enumeration_markdown(enumeration, enumerations, classes):
 
     return content
 
-def process_doclets(data, output_dir):
+def process_doclets(data, output_dir, editor_name):
     classes = {}
     classes_props = {}
     enumerations = []
+    editor_dir =  os.path.join(output_dir, editor_name)
 
     for doclet in data:
         if doclet['kind'] == 'class':
@@ -217,7 +220,7 @@ def process_doclets(data, output_dir):
 
     # Process classes
     for class_name, methods in classes.items():
-        class_dir = os.path.join(output_dir, class_name)
+        class_dir = os.path.join(editor_dir, class_name)
         methods_dir = os.path.join(class_dir, 'Methods')
         os.makedirs(methods_dir, exist_ok=True)
 
@@ -227,16 +230,22 @@ def process_doclets(data, output_dir):
 
         # Write method files
         for method in methods:
+            method_file_path = os.path.join(methods_dir, f"{method['name']}.md")
             method_content = generate_method_markdown(method, enumerations, classes)
-            write_markdown_file(os.path.join(methods_dir, f"{method['name']}.md"), method_content)
+            write_markdown_file(method_file_path, method_content)
+            if not method.get('example', ''):
+                missing_examples.append(os.path.relpath(method_file_path, output_dir))
 
     # Process enumerations
-    enum_dir = os.path.join(output_dir, 'Enumeration')
+    enum_dir = os.path.join(editor_dir, 'Enumeration')
     os.makedirs(enum_dir, exist_ok=True)
 
     for enum in enumerations:
+        enum_file_path = os.path.join(enum_dir, f"{enum['name']}.md")
         enum_content = generate_enumeration_markdown(enum, enumerations, classes)
-        write_markdown_file(os.path.join(enum_dir, f"{enum['name']}.md"), enum_content)
+        write_markdown_file(enum_file_path, enum_content)
+        if not enum.get('example', ''):
+            missing_examples.append(os.path.relpath(enum_file_path, output_dir))
 
 def generate(output_dir):
     print('Generating Markdown documentation...')
@@ -247,7 +256,7 @@ def generate(output_dir):
         os.makedirs(output_dir + f'/{editor_name.title()}', exist_ok=True)
 
         data = load_json(input_file)
-        process_doclets(data, output_dir + f'/{editor_name}')
+        process_doclets(data, output_dir, editor_name.title())
     
     shutil.rmtree(output_dir + 'tmp_json')
     print('Done')
@@ -262,5 +271,7 @@ if __name__ == "__main__":
         default="../../../../office-js-api/"  # Default value
     )
     args = parser.parse_args()
-    
     generate(args.destination)
+    print("START_MISSING_EXAMPLES")
+    print(",".join(missing_examples))
+    print("END_MISSING_EXAMPLES")

scripts/sdkjs_common/jsdoc/generate_docs_plugins_json.py

@@ -16,15 +16,9 @@ configs = [
 root = '../../../..'
 
 def generate(output_dir, md=False):
-    missing_examples_file = f'{output_dir}/missing_examples.txt'
-
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
 
-    # Recreate missing_examples.txt file
-    with open(missing_examples_file, 'w', encoding='utf-8') as f:
-        f.write('')
-
     # Generate JSON documentation
     for config in configs:
         editor_name = config.split('/')[-1].replace('.json', '')
@@ -73,11 +67,6 @@ def generate(output_dir, md=False):
                             if "forms" == document_type:
                                 document_type = "pdf"
                             doclet['description'] = doclet['description'] + f'\n\n## Try it\n\n ```js document-builder={{"documentType": "{document_type}"}}\n{code_content}\n```'
-                        
-                    else:
-                        # Record missing examples in missing_examples.txt
-                        with open(missing_examples_file, 'a', encoding='utf-8') as missing_file:
-                            missing_file.write(f"{file_path}\n")
         
         # Write the modified JSON file back
         with open(output_file, 'w', encoding='utf-8') as f:
@@ -96,25 +85,6 @@ def remove_js_comments(text):
     # Remove multi-line comments, leaving text after /*
     text = re.sub(r'/\*\s*|\s*\*/', '', text, flags=re.DOTALL)
     return text.strip()
-
-def get_current_branch(path):
-    try:
-        # Navigate to the specified directory and get the current branch name
-        result = subprocess.run(
-            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
-            cwd=path,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True
-        )
-        if result.returncode == 0:
-            return result.stdout.strip()
-        else:
-            print(f"Error: {result.stderr}")
-            return None
-    except Exception as e:
-        print(f"Exception: {e}")
-        return None
     
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Generate documentation")
@@ -123,15 +93,7 @@ if __name__ == "__main__":
         type=str, 
         help="Destination directory for the generated documentation",
         nargs='?',  # Indicates the argument is optional
-        default=f"{root}/document-builder-declarations/document-builder-plugin"  # Default value
+        default=f"{root}/office-js-api-declarations/office-js-api-plugins"
     )
     args = parser.parse_args()
-    
-    branch_name = get_current_branch(f"{root}/sdkjs")
-    if branch_name:
-        index_last_name = branch_name.rfind("/")
-        if -1 != index_last_name:
-            branch_name = branch_name[index_last_name + 1:]
-        args.destination = f"{args.destination}/{branch_name}"
-    
     generate(args.destination)

sln.json

@@ -27,7 +27,6 @@
     "core/DocxRenderer/DocxRenderer.pro",   
 
     "core/DesktopEditor/doctrenderer/doctrenderer.pro",
-    "core/DesktopEditor/doctrenderer/docbuilder.python/src/docbuilder_func_lib.pro",
 
     "[!no_x2t]core/OOXML/Projects/Linux/DocxFormatLib/DocxFormatLib.pro",
     "[!no_x2t]core/OOXML/Projects/Linux/PPTXFormatLib/PPTXFormatLib.pro",
@@ -66,7 +65,8 @@
   ],
 
   "builder" : [
-    "core"
+    "core",
+    "core/DesktopEditor/doctrenderer/docbuilder.python/src/docbuilder_func_lib.pro"
   ],
 
   "server" : [