Merge pull request 'hotfix/v8.3.1' (#49) from hotfix/v8.3.1 into develop

Reviewed-on: https://git.onlyoffice.com/ONLYOFFICE/build_tools/pulls/49

scripts/core_common/modules/v8_89.py

@@ -23,6 +23,9 @@ def change_bootstrap():
 
   base.replaceInFile("./depot_tools/bootstrap/bootstrap.py", 
     "raise subprocess.CalledProcessError(proc.returncode, argv, None)", "return")
+
+  base.replaceInFile("./depot_tools/bootstrap/bootstrap.py", 
+    "    _win_git_bootstrap_config()", "    #_win_git_bootstrap_config()")
   
   base.writeFile("./depot_tools/bootstrap/manifest.txt", content)
   return

scripts/deploy_desktop.py

@@ -184,6 +184,7 @@ def make():
         base.copy_file(git_dir + "/desktop-apps/win-linux/extras/projicons/" + apps_postfix + "/projicons.exe", root_dir + "/DesktopEditors.exe")
         if not isWindowsXP:
           base.copy_file(git_dir + "/desktop-apps/win-linux/extras/update-daemon/" + apps_postfix + "/updatesvc.exe", root_dir + "/updatesvc.exe")
+        else:
           base.copy_file(git_dir + "/desktop-apps/win-linux/extras/online-installer/" + apps_postfix + "/online-installer.exe", root_dir + "/online-installer.exe")
         base.copy_file(git_dir + "/desktop-apps/win-linux/" + apps_postfix + "/DesktopEditors.exe", root_dir + "/editors.exe")
         base.copy_file(git_dir + "/desktop-apps/win-linux/res/icons/desktopeditors.ico", root_dir + "/app.ico")

scripts/develop/dependence.py

@@ -190,7 +190,7 @@ def check_nodejs():
   nodejs_cur_version_major = int(nodejs_version.split('.')[0][1:])
   nodejs_cur_version_minor = int(nodejs_version.split('.')[1])
   print('Installed Node.js version: ' + nodejs_version[1:])
-  nodejs_min_version = '22'
+  nodejs_min_version = '18'
   nodejs_min_version_minor  = 0
   major_minor_min_version = nodejs_min_version.split('.')
   nodejs_min_version_major = int(major_minor_min_version[0])
@@ -214,7 +214,7 @@ def check_nodejs():
       isNeedReinstall = True
 
   if (True == isNeedReinstall):
-    print('Installed Node.js version must be 22 or higher.')
+    print('Installed Node.js version must be 18 or higher.')
     if (host_platform == 'windows'):
       dependence.append_uninstall('Node.js')
       dependence.append_install('Node.js')
@@ -950,7 +950,7 @@ def install_postgresql():
   return code
 
 def install_nodejs():
-  os.system('curl -sSL https://deb.nodesource.com/setup_22.x | sudo -E bash -')
+  os.system('curl -sSL https://deb.nodesource.com/setup_18.x | sudo -E bash -')
   base.print_info("Install node.js...")
   install_command = 'yes | sudo apt install nodejs'
   print(install_command)
@@ -959,7 +959,7 @@ def install_nodejs():
 downloads_list = {
   'Windows': {
     'Git': 'https://github.com/git-for-windows/git/releases/download/v2.29.0.windows.1/Git-2.29.0-64-bit.exe',
-    'Node.js': 'https://nodejs.org/dist/v22.13.1/node-v22.13.1-x64.msi',
+    'Node.js': 'https://nodejs.org/dist/v18.17.1/node-v18.17.1-x64.msi',
     'Java': 'https://aka.ms/download-jdk/microsoft-jdk-11.0.18-windows-x64.msi',
     'RabbitMQ': 'https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.8.9/rabbitmq-server-3.8.9.exe',
     'Erlang': 'http://erlang.org/download/otp_win64_23.1.exe',

scripts/package_desktop.py

@@ -178,7 +178,7 @@ def make_advinst():
   return
 
 def make_online():
-  if not common.platform in ["windows_x64", "windows_x86"]:
+  if not common.platform in ["windows_x86_xp"]:
     return
   online_file = "%s-%s-%s.exe" % ("OnlineInstaller", package_version, suffix)
   ret = utils.is_file(online_file)

scripts/sdkjs_common/jsdoc/generate_docs_md.py

@@ -153,7 +153,7 @@ def generate_data_types_markdown(types, enumerations, classes, root='../../'):
     linked = [link_if_known(ts_t) for ts_t in converted]
 
     # Join them with " | "
-    param_types_md = ' | '.join(linked)
+    param_types_md = r' \| '.join(linked)
 
     # If there's still leftover angle brackets for generics, gently escape or link them
     # e.g. "Object.<string, number>" => "Object.&lt;string, number&gt;"
@@ -178,7 +178,7 @@ def generate_class_markdown(class_name, methods, properties, enumerations, class
     # Escape just before returning
     return escape_text_outside_code_blocks(content)
 
-def generate_method_markdown(method, enumerations, classes):
+def generate_method_markdown(method, enumerations, classes, example_editor_name):
     method_name = method['name']
     description = method.get('description', 'No description provided.')
     description = correct_description(description)
@@ -227,11 +227,11 @@ def generate_method_markdown(method, enumerations, classes):
         if '```js' in example:
             comment, code = example.split('```js', 1)
             comment = remove_js_comments(comment)
-            content += f"\n\n## Example\n\n{comment}\n\n```javascript\n{code.strip()}\n"
+            content += f"\n\n## Example\n\n{comment}\n\n```javascript {example_editor_name}\n{code.strip()}\n"
         else:
             # If there's no triple-backtick structure, just show it as code
             cleaned_example = remove_js_comments(example)
-            content += f"\n\n## Example\n\n```javascript\n{cleaned_example}\n```\n"
+            content += f"\n\n## Example\n\n```javascript {example_editor_name}\n{cleaned_example}\n```\n"
 
     return escape_text_outside_code_blocks(content)
 
@@ -254,7 +254,7 @@ def generate_properties_markdown(properties, enumerations, classes, root='../'):
     # Escape outside code blocks
     return escape_text_outside_code_blocks(content)
 
-def generate_enumeration_markdown(enumeration, enumerations, classes):
+def generate_enumeration_markdown(enumeration, enumerations, classes, example_editor_name):
     enum_name = enumeration['name']
     description = enumeration.get('description', 'No description provided.')
     description = correct_description(description)
@@ -300,11 +300,11 @@ def generate_enumeration_markdown(enumeration, enumerations, classes):
         if '```js' in example:
             comment, code = example.split('```js', 1)
             comment = remove_js_comments(comment)
-            content += f"\n\n## Example\n\n{comment}\n\n```javascript\n{code.strip()}\n"
+            content += f"\n\n## Example\n\n{comment}\n\n```javascript {example_editor_name}\n{code.strip()}\n"
         else:
             # If there's no triple-backtick structure
             cleaned_example = remove_js_comments(example)
-            content += f"\n\n## Example\n\n```javascript\n{cleaned_example}\n```\n"
+            content += f"\n\n## Example\n\n```javascript {example_editor_name}\n{cleaned_example}\n```\n"
 
     return escape_text_outside_code_blocks(content)
 
@@ -313,6 +313,16 @@ def process_doclets(data, output_dir, editor_name):
     classes_props = {}
     enumerations = []
     editor_dir =  os.path.join(output_dir, editor_name)
+    example_editor_name = 'editor-'
+    
+    if editor_name == 'Word':
+        example_editor_name += 'docx'
+    elif editor_name == 'Forms':
+        example_editor_name += 'pdf'
+    elif editor_name == 'Slide':
+        example_editor_name += 'pptx'
+    elif editor_name == 'Cell':
+        example_editor_name += 'xlsx'
 
     for doclet in data:
         if doclet['kind'] == 'class':
@@ -347,7 +357,7 @@ def process_doclets(data, output_dir, editor_name):
         # Write method files
         for method in methods:
             method_file_path = os.path.join(methods_dir, f"{method['name']}.md")
-            method_content = generate_method_markdown(method, enumerations, classes)
+            method_content = generate_method_markdown(method, enumerations, classes, example_editor_name)
             write_markdown_file(method_file_path, method_content)
 
             if not method.get('example', ''):
@@ -359,7 +369,7 @@ def process_doclets(data, output_dir, editor_name):
 
     for enum in enumerations:
         enum_file_path = os.path.join(enum_dir, f"{enum['name']}.md")
-        enum_content = generate_enumeration_markdown(enum, enumerations, classes)
+        enum_content = generate_enumeration_markdown(enum, enumerations, classes, example_editor_name)
         if enum_content is None:
             continue
 

scripts/sdkjs_common/jsdoc/generate_docs_plugins_md.py

@@ -153,7 +153,7 @@ def generate_data_types_markdown(types, enumerations, classes, root='../../'):
     linked = [link_if_known(ts_t) for ts_t in converted]
 
     # Join them with " | "
-    param_types_md = ' | '.join(linked)
+    param_types_md = r' \| '.join(linked)
 
     # If there's still leftover angle brackets for generics, gently escape or link them
     # e.g. "Object.<string, number>" => "Object.&lt;string, number&gt;"
@@ -178,7 +178,7 @@ def generate_class_markdown(class_name, methods, properties, enumerations, class
     # Escape just before returning
     return escape_text_outside_code_blocks(content)
 
-def generate_method_markdown(method, enumerations, classes):
+def generate_method_markdown(method, enumerations, classes, example_editor_name):
     """
     Generates Markdown for a method doclet, relying only on `method['examples']`
     (array of strings). Ignores any single `method['example']` field.
@@ -247,12 +247,12 @@ def generate_method_markdown(method, enumerations, classes):
                 if len(examples) > 1:
                     content += f"**Example {i}:**\n\n{comment}\n\n"
                 
-                content += f"```javascript\n{code}\n```\n"
+                content += f"```javascript {example_editor_name}\n{code}\n```\n"
             else:
                 if len(examples) > 1:
                     content += f"**Example {i}:**\n\n{comment}\n\n"
                 # No special fences, just show as code
-                content += f"```javascript\n{cleaned_example}\n```\n"
+                content += f"```javascript {example_editor_name}\n{cleaned_example}\n```\n"
 
     return escape_text_outside_code_blocks(content)
 
@@ -275,7 +275,7 @@ def generate_properties_markdown(properties, enumerations, classes, root='../'):
     # Escape outside code blocks
     return escape_text_outside_code_blocks(content)
 
-def generate_enumeration_markdown(enumeration, enumerations, classes):
+def generate_enumeration_markdown(enumeration, enumerations, classes, example_editor_name):
     """
     Generates Markdown documentation for a 'typedef' doclet.
     This version only works with `enumeration['examples']` (an array of strings),
@@ -354,12 +354,12 @@ def generate_enumeration_markdown(enumeration, enumerations, classes):
                 if len(examples) > 1:
                     content += f"**Example {i}:**\n\n{comment}\n\n"
                 
-                content += f"```javascript\n{code}\n```\n"
+                content += f"```javascript {example_editor_name}\n{code}\n```\n"
             else:
                 if len(examples) > 1:
                     content += f"**Example {i}:**\n\n{comment}\n\n"
                 # No special fences, just show as code
-                content += f"```javascript\n{cleaned_example}\n```\n"
+                content += f"```javascript {example_editor_name}\n{cleaned_example}\n```\n"
 
     return escape_text_outside_code_blocks(content)
 
@@ -368,6 +368,16 @@ def process_doclets(data, output_dir, editor_name):
     classes_props = {}
     enumerations = []
     editor_dir =  os.path.join(output_dir, editor_name)
+    example_editor_name = 'editor-'
+    
+    if editor_name == 'Word':
+        example_editor_name += 'docx'
+    elif editor_name == 'Forms':
+        example_editor_name += 'pdf'
+    elif editor_name == 'Slide':
+        example_editor_name += 'pptx'
+    elif editor_name == 'Cell':
+        example_editor_name += 'xlsx'
 
     for doclet in data:
         if doclet['kind'] == 'class':
@@ -402,7 +412,7 @@ def process_doclets(data, output_dir, editor_name):
         # Write method files
         for method in methods:
             method_file_path = os.path.join(methods_dir, f"{method['name']}.md")
-            method_content = generate_method_markdown(method, enumerations, classes)
+            method_content = generate_method_markdown(method, enumerations, classes, example_editor_name)
             write_markdown_file(method_file_path, method_content)
 
             if not method.get('examples', ''):
@@ -414,7 +424,7 @@ def process_doclets(data, output_dir, editor_name):
 
     for enum in enumerations:
         enum_file_path = os.path.join(enum_dir, f"{enum['name']}.md")
-        enum_content = generate_enumeration_markdown(enum, enumerations, classes)
+        enum_content = generate_enumeration_markdown(enum, enumerations, classes, example_editor_name)
         if enum_content is None:
             continue
 

scripts/sdkjs_common/jsdoc/generate_jsonl_dataset.py

@@ -0,0 +1,248 @@
+import os
+import json
+import re
+import shutil
+import argparse
+import generate_docs_json
+from datetime import datetime
+
+# Configuration files
+editors = [
+    "word",
+    "cell",
+    "slide",
+    "forms"
+]
+
+root = '../../../..'
+missing_examples = []
+
+def load_json(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        return json.load(f)
+
+def read_file_content(file_path):
+    try:
+        with open(file_path, encoding='utf-8') as f:
+            return f.read()
+    except Exception as e:
+        missing_examples.append(file_path)
+        # print(f"Failed to open file {file_path}: {e}")
+        return ""
+
+def extract_js_comments_as_text(text):
+    # Extract single-line comments (after //)
+    single_line_comments = re.findall(r'^\s*//(.*)$', text, flags=re.MULTILINE)
+    # Extract multi-line comments (between /* and */)
+    multi_line_comments = re.findall(r'/\*(.*?)\*/', text, flags=re.DOTALL)
+    # Combine all found comments into a single list
+    all_comments = single_line_comments + multi_line_comments
+    # Join comments into a single text, separated by a space
+    return " ".join(comment.strip() for comment in all_comments if comment.strip())
+
+def extract_examples_blocks(content: str):
+    blocks = []
+    current_block = {"comments": [], "code": []}
+    in_comment_section = True  # Collect comments until code appears
+    current_comment_group = []  # Accumulate lines of the current comment
+
+    for line in content.splitlines():
+        stripped = line.strip()
+        if not stripped:
+            # Empty line
+            if in_comment_section and current_comment_group:
+                # Finish the current comment group
+                comment_text = " ".join(current_comment_group)
+                current_block["comments"].append(comment_text)
+                current_comment_group = []
+            elif not in_comment_section:
+                # Empty line in the code – keep it as is
+                current_block["code"].append(line)
+            continue
+
+        if stripped.startswith("//"):
+            if in_comment_section:
+                # Remove comment marker and extra spaces
+                current_comment_group.append(extract_js_comments_as_text(stripped))
+            else:
+                # Comment after code starts – finish the current block and start a new one
+                blocks.append({
+                    "comments": current_block["comments"],
+                    "code": "\n".join(current_block["code"]).rstrip()
+                })
+                current_block = {"comments": [], "code": []}
+                in_comment_section = True
+                # Start a new comment group with the current line
+                current_comment_group = [stripped[2:].strip()]
+        else:
+            # Code line
+            if in_comment_section:
+                if current_comment_group:
+                    comment_text = " ".join(current_comment_group)
+                    current_block["comments"].append(comment_text)
+                    current_comment_group = []
+                in_comment_section = False
+            current_block["code"].append(line)
+
+    # Finalize any remaining comment group
+    if in_comment_section and current_comment_group:
+        comment_text = " ".join(current_comment_group)
+        current_block["comments"].append(comment_text)
+    # Save the last block if it's not empty
+    if current_block["comments"] or current_block["code"]:
+        blocks.append({
+            "comments": current_block["comments"],
+            "code": "\n".join(current_block["code"]).rstrip()
+        })
+
+    return blocks
+
+def extract_examples_blocks_temp(content: str):
+    lines = content.splitlines()
+    comment_blocks = []
+    current_group = []
+    first_code_index = None
+
+    for i, line in enumerate(lines):
+        stripped = line.strip()
+        if not stripped:
+            if current_group:
+                comment_blocks.append(" ".join(current_group))
+                current_group = []
+            continue
+        if stripped.startswith("//"):
+            current_group.append(stripped[2:].strip())
+        else:
+            if current_group:
+                comment_blocks.append(" ".join(current_group))
+                current_group = []
+            first_code_index = i
+            break
+
+    code_part = ""
+    if first_code_index is not None:
+        code_part = "\n".join(lines[first_code_index:]).rstrip()
+
+    return [{"comments": comment_blocks, "code": code_part}]
+
+def create_entry(system_message, user_message, assistant_message, model):
+    entry = {
+        "created_at": datetime.now().isoformat(" "),
+        "messages": [
+            {"role": "system", "content": system_message},
+            {"role": "user", "content": user_message},
+            {"role": "assistant", "content": assistant_message}
+        ],
+        "recommended": False,
+        "upvoted": True
+    }
+
+    if model is not "":
+        entry["model"] = model
+
+    return entry 
+
+def process_doclets(doclets, output_entries, editor_name, model):
+    for doclet in doclets:
+        kind = doclet.get("kind", "").lower()
+        see = doclet.get("see", [])
+        
+        # The "see" field must always be present
+        if not see:
+            continue
+        
+        # Processing based on the "kind" value
+        if kind == "function":
+            method_name = doclet.get("name", "")
+            memberof = doclet.get("memberof", "")
+            # Functions must have both "name" (method_name) and "memberof" fields filled
+            if not (method_name and memberof):
+                continue
+            system_message = (
+                f"You act as an API expert for Onlyoffice {editor_name.title()} editor. "
+                f"This task is an example for the function {method_name} in the class {memberof}."
+            )
+            default_user_message = f"How do I use the method {method_name} of {memberof} class?"
+            
+        elif kind == "class":
+            class_name = doclet.get("name", "")
+            system_message = (
+                f"You act as an API expert for Onlyoffice {editor_name.title()} editor. "
+                f"This task is an example for the class {class_name}."
+            )
+            default_user_message = f"How do I instantiate or work with the class {class_name}?"
+            
+        elif kind == "typedef":
+            typedef_name = doclet.get("name", "")
+            system_message = (
+                f"You act as an API expert for Onlyoffice {editor_name.title()} editor. "
+                f"This task is an example for the typedef {typedef_name}"
+            )
+            default_user_message = f"How do I use the typedef {typedef_name}?"
+            
+        else:
+            continue
+        
+        # Read the content of the first file listed in the "see" field
+        content = read_file_content(f'{root}/{see[0]}')
+        if content == "":
+            continue
+        
+        # now use only first block cause there is bad comments in examples
+        blocks = extract_examples_blocks_temp(content)
+        
+        for block in blocks:
+            assistant_message = block['code']
+
+            # default entry
+            output_entries.append(create_entry(system_message, default_user_message, assistant_message, model))
+
+            # If the file content contains comments, create a separate entry for each one
+            for comment in block['comments']:
+                output_entries.append(create_entry(system_message, comment, assistant_message, model))
+            
+def generate(output_dir, model):
+    print('Generating documentation JSONL dataset...')
+    
+    shutil.rmtree(output_dir, ignore_errors=True)
+    os.makedirs(output_dir)
+
+    generate_docs_json.generate(f'{output_dir}/tmp_json')
+    
+    output_entries = []
+    output_filename = "dataset.jsonl"
+
+    for editor_name in editors:
+        input_file = os.path.join(f'{output_dir}/tmp_json', editor_name + ".json")
+        doclets = load_json(input_file)
+        process_doclets(doclets, output_entries, editor_name, model)
+    
+    with open(f'{output_dir}/{output_filename}', "w", encoding="utf-8") as out_file:
+        for entry in output_entries:
+            out_file.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+    shutil.rmtree(f'{output_dir}/tmp_json')
+    print('Done')
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Generate documentation JSONL dataset")
+    parser.add_argument(
+        "destination", 
+        type=str, 
+        help="Destination directory for the generated documentation",
+        nargs='?',  # Indicates the argument is optional
+        default="../../../../office-js-api/dataset"  # Default value
+    )
+    parser.add_argument(
+        "model", 
+        type=str, 
+        help="Type of model",
+        nargs='?',  # Indicates the argument is optional
+        default=""  # Default value
+    )
+    args = parser.parse_args()
+
+    generate(args.destination, args.model)
+    print("START_MISSING_EXAMPLES")
+    print(",".join(missing_examples))
+    print("END_MISSING_EXAMPLES")

sln.json

@@ -93,7 +93,7 @@
     
     "[win]desktop-apps/win-linux/extras/projicons/ProjIcons.pro",
     "[win,!win_xp]desktop-apps/win-linux/extras/update-daemon/UpdateDaemon.pro",
-    "[win,!win_xp]desktop-apps/win-linux/extras/online-installer/OnlineInstaller.pro"
+    "[win_xp]desktop-apps/win-linux/extras/online-installer/OnlineInstaller.pro"
   ],
 
   "mobile" : [

tools/linux/deps.py

@@ -67,7 +67,7 @@ def install_deps():
     print("OK")
     base.cmd("sudo", ["apt-get", "-y", "install", "npm", "yarn"], True)
   base.cmd("sudo", ["npm", "install", "-g", "grunt-cli"])
-  base.cmd("sudo", ["npm", "install", "-g", "@yao-pkg/pkg"])
+  base.cmd("sudo", ["npm", "install", "-g", "pkg"])
 
   # java
   java_error = base.cmd("sudo", ["apt-get", "-y", "install", "openjdk-11-jdk"], True)

version

@@ -1 +1 @@
-8.2.2
+8.3.0