From 4716b633e2b55aa8a2a145d0057a450eb525a36f Mon Sep 17 00:00:00 2001 From: Flexus <37239071+EnergyFlexus@users.noreply.github.com> Date: Wed, 14 Sep 2022 18:01:46 +0300 Subject: [PATCH 01/17] Hyphen test add --- .gitignore | 1 + Common/3dParty/hyphen/.gitignore | 5 ++ Common/3dParty/hyphen/test/main.cpp | 91 +++++++++++++++++++++++++++++ Common/3dParty/hyphen/test/test.pro | 25 ++++++++ 4 files changed, 122 insertions(+) create mode 100644 Common/3dParty/hyphen/.gitignore create mode 100644 Common/3dParty/hyphen/test/main.cpp create mode 100644 Common/3dParty/hyphen/test/test.pro diff --git a/.gitignore b/.gitignore index 8cb8cb83c4..edd2496b41 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ Thumbs.db *.opendb DesktopEditor/fontengine/js/common/freetype-2.10.4 +*_resource.rc diff --git a/Common/3dParty/hyphen/.gitignore b/Common/3dParty/hyphen/.gitignore new file mode 100644 index 0000000000..476026af48 --- /dev/null +++ b/Common/3dParty/hyphen/.gitignore @@ -0,0 +1,5 @@ +hyphen +test.pro.user +*.dic +msvc_make.bat +build-test-* \ No newline at end of file diff --git a/Common/3dParty/hyphen/test/main.cpp b/Common/3dParty/hyphen/test/main.cpp new file mode 100644 index 0000000000..2819ac9c50 --- /dev/null +++ b/Common/3dParty/hyphen/test/main.cpp @@ -0,0 +1,91 @@ +#include +#include + +#include "./../hyphen/hyphen.h" + +int main(int argc, char *argv[]) +{ + HyphenDict *dict; + + std::string dict_filename = PRO_DIR; + std::string words_filename = PRO_DIR; + std::string result_filename = PRO_DIR; + + // set your filenames here + dict_filename += "hyph_ru_RU.dic"; + words_filename += "words.txt"; + result_filename += "result.txt"; + + // load the hyphenation dictionary + dict = hnj_hyphen_load(dict_filename.c_str()); + + std::ifstream fin(words_filename); + if(!fin.is_open()) + { + std::cerr << "could not open " << words_filename << "!" << std::endl; + return -1; + } + + std::ofstream fout(result_filename); + if(!fout.is_open()) + { + std::cerr << "could not open " << result_filename << "!" << std::endl; + return -1; + } + + while(!fin.eof()) + { + char **rep = NULL; + int *pos = NULL; + int *cut = NULL; + + std::string word; + + fin >> word; + int n = word.size(); + char *hword = new char[n * 2]; + char *hyphens = new char[n + 5]; + + /** + * @brief + * input data: + * + * word: input word + * word_size: byte length of the input word + * hyphens: allocated character buffer (size = word_size + 5) + * hyphenated_word: allocated character buffer (size ~ word_size * 2) or NULL + * rep, pos, cut: pointers (point to the allocated and _zeroed_ buffers + * (size=word_size) or with NULL value) or NULL + * + * output data: + * + * hyphens: hyphenation vector (hyphenation points signed with odd numbers). + * hyphenated_word: hyphenated input word (hyphens signed with `='). + * optional (NULL input). + * rep: NULL (only standard hyph.), or replacements (hyphenation points + * signed with `=' in replacements). + * pos: NULL, or difference of the actual position and the beginning + * positions of the change in input words. + * cut: NULL, or counts of the removed characters of the original words + * at hyphenation. + * + * Note: rep, pos, cut are complementary arrays to the hyphens, indexed with the + * character positions of the input word. + */ + hnj_hyphen_hyphenate2(dict, word.c_str(), n, hyphens, hword, &rep, &pos, &cut); + + fout << hword << ' '; + + delete[] hword; + delete[] hyphens; + } + fin.close(); + fout.close(); +} + + + + + + + diff --git a/Common/3dParty/hyphen/test/test.pro b/Common/3dParty/hyphen/test/test.pro new file mode 100644 index 0000000000..787a92ef45 --- /dev/null +++ b/Common/3dParty/hyphen/test/test.pro @@ -0,0 +1,25 @@ +QT -= core +QT -= gui + +TARGET = test +CONFIG += console +CONFIG -= app_bundle + +TEMPLATE = app + +CORE_ROOT_DIR = $$PWD/../../../../../core +PWD_ROOT_DIR = $$PWD + +include($$CORE_ROOT_DIR/Common/base.pri) + +INCLUDEPATH += $$PWD_ROOT_DIR/../hyphen + +DEFINES += PRO_DIR=\\\"$$PWD/\\\" + +HEADERS += $$PWD_ROOT_DIR/../hyphen/hyphen.h +HEADERS += $$PWD_ROOT_DIR/../hyphen/hnjalloc.h + +SOURCES += $$PWD_ROOT_DIR/../hyphen/hyphen.c +SOURCES += $$PWD_ROOT_DIR/../hyphen/hnjalloc.c + +SOURCES += main.cpp From 34a2091dfc18177f18744c6f1563090937c761f8 Mon Sep 17 00:00:00 2001 From: Flexus <37239071+EnergyFlexus@users.noreply.github.com> Date: Thu, 15 Sep 2022 15:14:01 +0300 Subject: [PATCH 02/17] Hyphen js+wasm test add --- Common/3dParty/hyphen/.gitignore | 5 +- Common/3dParty/hyphen/js/index.html | 16 +++++ Common/3dParty/hyphen/js/main.js | 72 +++++++++++++++++++ .../hyphen/js/src/exported_functions.cpp | 25 +++++++ .../hyphen/js/src/exported_functions.h | 16 +++++ Common/3dParty/hyphen/js/styles.css | 10 +++ Common/3dParty/hyphen/test/main.cpp | 34 ++++----- 7 files changed, 160 insertions(+), 18 deletions(-) create mode 100644 Common/3dParty/hyphen/js/index.html create mode 100644 Common/3dParty/hyphen/js/main.js create mode 100644 Common/3dParty/hyphen/js/src/exported_functions.cpp create mode 100644 Common/3dParty/hyphen/js/src/exported_functions.h create mode 100644 Common/3dParty/hyphen/js/styles.css diff --git a/Common/3dParty/hyphen/.gitignore b/Common/3dParty/hyphen/.gitignore index 476026af48..7e727c1ce9 100644 --- a/Common/3dParty/hyphen/.gitignore +++ b/Common/3dParty/hyphen/.gitignore @@ -2,4 +2,7 @@ hyphen test.pro.user *.dic msvc_make.bat -build-test-* \ No newline at end of file +build-test-* +*.wasm +hyphen.js +hyphen.data \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/index.html b/Common/3dParty/hyphen/js/index.html new file mode 100644 index 0000000000..898541c53e --- /dev/null +++ b/Common/3dParty/hyphen/js/index.html @@ -0,0 +1,16 @@ + + + + + test + + + +
+ + +
+ + + + \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/main.js b/Common/3dParty/hyphen/js/main.js new file mode 100644 index 0000000000..3fb8cf3360 --- /dev/null +++ b/Common/3dParty/hyphen/js/main.js @@ -0,0 +1,72 @@ +/** + * @param {Module} module + * @param {Number} length + * @return {Number} + * Allocate memory for wasm, returns pointer + */ +function allocateMemory(module, length) { + const ptr = module._malloc(length); + return ptr; +} + +/** + * @param {Module} module + * @param {Number} ptr + * @return {void} + * Free memory + */ +function freeMemory(module, ptr) { + module._free(ptr); +} + +/** + * @param {Module} module + * @param {Uint8Array} data + * @param {Number} ptr + * @return {void} + * Fill memory + */ +function setMemory(module, data, ptr) { + module.HEAP8.set(data, ptr); +} + +/** + * @type {HTMLTextAreaElement} + */ +var textarea = document.getElementById("textarea"); +var form = document.querySelector("form"); + +function hyphenate(word) +{ + var len = word.length; + var word_size = 4 * len + 1; + var hyphen_size = word_size + 5; + var hword_size = word_size * 2; + + var pword = allocateMemory(Module, word_size); + var phyphens = allocateMemory(Module, hyphen_size); + var phword = allocateMemory(Module, hword_size); + + Module.stringToUTF8(word, pword, word_size); + Module._hyphenate(pword, phyphens, phword) + + var hword = Module.UTF8ToString(phword); + + freeMemory(Module, pword); + freeMemory(Module, phyphens); + freeMemory(Module, phword); + + return hword; +} + +form.onsubmit = function(event) { + event.preventDefault(); + var text = textarea.value.split("\n").join(" ").split(" "); + Module._load_dictionary(); + + for(var i = 0; i < text.length; i++) { + console.log(hyphenate(text[i])); + } + Module._free_dictionary(); +} + diff --git a/Common/3dParty/hyphen/js/src/exported_functions.cpp b/Common/3dParty/hyphen/js/src/exported_functions.cpp new file mode 100644 index 0000000000..4dd1826c57 --- /dev/null +++ b/Common/3dParty/hyphen/js/src/exported_functions.cpp @@ -0,0 +1,25 @@ +#include "exported_functions.h" +#include + +HyphenDict *dict; + +void load_dictionary() +{ + dict = hnj_hyphen_load("dictionaries/hyph_ru_RU.dic"); +} +void free_dictionary() +{ + hnj_hyphen_free(dict); +} + +void hyphenate(const char *word, char *hyphens, char *hword) +{ + int n = strlen(word); + + char **rep = NULL; + int *pos = NULL; + int *cut = NULL; + + hnj_hyphen_hyphenate2(dict, word, n, hyphens, hword, &rep, &pos, &cut); +} +// emcc src/exported_functions.cpp ../hyphen/hyphen.c ../hyphen/hnjalloc.c -o deploy/effects.js -sEXPORTED_FUNCTIONS='_malloc','_free','_hyphenate','_load_dictionary','_free_dictionary' -sEXPORTED_RUNTIME_METHODS='stringToUTF8','UTF8ToString' --preload-file dictionaries \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/src/exported_functions.h b/Common/3dParty/hyphen/js/src/exported_functions.h new file mode 100644 index 0000000000..6d5cb0b84d --- /dev/null +++ b/Common/3dParty/hyphen/js/src/exported_functions.h @@ -0,0 +1,16 @@ +#include + +#include "./../../hyphen/hyphen.h" + +#ifdef __cplusplus +extern "C" +{ +#endif // __cplusplus + +void load_dictionary(); +void free_dictionary(); +void hyphenate(const char *word, char *hyphens, char *hword); + +#ifdef __cplusplus +} +#endif // __cplusplus \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/styles.css b/Common/3dParty/hyphen/js/styles.css new file mode 100644 index 0000000000..a521d17e8b --- /dev/null +++ b/Common/3dParty/hyphen/js/styles.css @@ -0,0 +1,10 @@ +button { + width: 60px; + height: 30px; +} + +#textarea{ + display: block; + width: 300px; + height: 300px; +} \ No newline at end of file diff --git a/Common/3dParty/hyphen/test/main.cpp b/Common/3dParty/hyphen/test/main.cpp index 2819ac9c50..b7232ce4b1 100644 --- a/Common/3dParty/hyphen/test/main.cpp +++ b/Common/3dParty/hyphen/test/main.cpp @@ -12,7 +12,7 @@ int main(int argc, char *argv[]) std::string result_filename = PRO_DIR; // set your filenames here - dict_filename += "hyph_ru_RU.dic"; + dict_filename += "hyph_en_US.dic"; words_filename += "words.txt"; result_filename += "result.txt"; @@ -50,27 +50,27 @@ int main(int argc, char *argv[]) * @brief * input data: * - * word: input word - * word_size: byte length of the input word - * hyphens: allocated character buffer (size = word_size + 5) - * hyphenated_word: allocated character buffer (size ~ word_size * 2) or NULL - * rep, pos, cut: pointers (point to the allocated and _zeroed_ buffers - * (size=word_size) or with NULL value) or NULL + * word: input word + * word_size: byte length of the input word + * hyphens: allocated character buffer (size = word_size + 5) + * hyphenated_word: allocated character buffer (size ~ word_size * 2) or NULL + * rep, pos, cut: pointers (point to the allocated and _zeroed_ buffers + * (size=word_size) or with NULL value) or NULL * * output data: * - * hyphens: hyphenation vector (hyphenation points signed with odd numbers). - * hyphenated_word: hyphenated input word (hyphens signed with `='). - * optional (NULL input). - * rep: NULL (only standard hyph.), or replacements (hyphenation points - * signed with `=' in replacements). - * pos: NULL, or difference of the actual position and the beginning - * positions of the change in input words. - * cut: NULL, or counts of the removed characters of the original words - * at hyphenation. + * hyphens: hyphenation vector (hyphenation points signed with odd numbers). + * hyphenated_word: hyphenated input word (hyphens signed with `='). + * optional (NULL input). + * rep: NULL (only standard hyph.), or replacements (hyphenation points + * signed with `=' in replacements). + * pos: NULL, or difference of the actual position and the beginning + * positions of the change in input words. + * cut: NULL, or counts of the removed characters of the original words + * at hyphenation. * * Note: rep, pos, cut are complementary arrays to the hyphens, indexed with the - * character positions of the input word. + * character positions of the input word. */ hnj_hyphen_hyphenate2(dict, word.c_str(), n, hyphens, hword, &rep, &pos, &cut); From b01dd7eefc2a1e086fc107fd03fd6dea5a571528 Mon Sep 17 00:00:00 2001 From: Flexus <37239071+EnergyFlexus@users.noreply.github.com> Date: Thu, 15 Sep 2022 18:08:08 +0300 Subject: [PATCH 03/17] Hyphen test update --- Common/3dParty/hyphen/js/main.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Common/3dParty/hyphen/js/main.js b/Common/3dParty/hyphen/js/main.js index 3fb8cf3360..4f98b7bf7b 100644 --- a/Common/3dParty/hyphen/js/main.js +++ b/Common/3dParty/hyphen/js/main.js @@ -51,12 +51,22 @@ function hyphenate(word) Module._hyphenate(pword, phyphens, phword) var hword = Module.UTF8ToString(phword); - + var hyphens = Module.UTF8ToString(phyphens); + + var positions = []; + for(var i = 0; i < hyphens.length; i++) { + // hyphenation vector has odd number in positions where hyphen is needed + if(Number(hyphens[i]) % 2 == 1) { + positions.push(i); + } + } + console.log(hword); + freeMemory(Module, pword); freeMemory(Module, phyphens); freeMemory(Module, phword); - return hword; + return positions; } form.onsubmit = function(event) { @@ -65,7 +75,7 @@ form.onsubmit = function(event) { Module._load_dictionary(); for(var i = 0; i < text.length; i++) { - console.log(hyphenate(text[i])); + console.log(hyphenate(text[i].toLowerCase())); } Module._free_dictionary(); } From 59cad91a2204d5c6fb0317b1b455f8c30c08f3f8 Mon Sep 17 00:00:00 2001 From: Alexey Date: Mon, 19 Sep 2022 16:23:37 +0300 Subject: [PATCH 04/17] HyphenUtils add --- Common/3dParty/hyphen/.gitignore | 3 +- Common/3dParty/hyphen/js/index.html | 1 + Common/3dParty/hyphen/js/main.js | 13 ++ .../hyphen/js/src/HyphenApplication.cpp | 1 + .../3dParty/hyphen/js/src/HyphenApplication.h | 25 +++ Common/3dParty/hyphen/js/src/HyphenUtils.cpp | 180 ++++++++++++++++++ .../hyphen/js/src/exported_functions.cpp | 37 +++- .../hyphen/js/src/exported_functions.h | 9 +- 8 files changed, 255 insertions(+), 14 deletions(-) create mode 100644 Common/3dParty/hyphen/js/src/HyphenApplication.cpp create mode 100644 Common/3dParty/hyphen/js/src/HyphenApplication.h create mode 100644 Common/3dParty/hyphen/js/src/HyphenUtils.cpp diff --git a/Common/3dParty/hyphen/.gitignore b/Common/3dParty/hyphen/.gitignore index 7e727c1ce9..bb01c12494 100644 --- a/Common/3dParty/hyphen/.gitignore +++ b/Common/3dParty/hyphen/.gitignore @@ -5,4 +5,5 @@ msvc_make.bat build-test-* *.wasm hyphen.js -hyphen.data \ No newline at end of file +hyphen.data +.vscode \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/index.html b/Common/3dParty/hyphen/js/index.html index 898541c53e..1b2c1978a0 100644 --- a/Common/3dParty/hyphen/js/index.html +++ b/Common/3dParty/hyphen/js/index.html @@ -9,6 +9,7 @@
+
diff --git a/Common/3dParty/hyphen/js/main.js b/Common/3dParty/hyphen/js/main.js index 4f98b7bf7b..1800062f87 100644 --- a/Common/3dParty/hyphen/js/main.js +++ b/Common/3dParty/hyphen/js/main.js @@ -36,6 +36,19 @@ function setMemory(module, data, ptr) { var textarea = document.getElementById("textarea"); var form = document.querySelector("form"); +var control = document.getElementById("dictionary-file"); +control.addEventListener("change", function(event) { + + var file = control.files[0]; + var reader = new FileReader(); + reader.readAsText(file); + + reader.onload = function() { + + }; + +}, false); + function hyphenate(word) { var len = word.length; diff --git a/Common/3dParty/hyphen/js/src/HyphenApplication.cpp b/Common/3dParty/hyphen/js/src/HyphenApplication.cpp new file mode 100644 index 0000000000..32a3c28470 --- /dev/null +++ b/Common/3dParty/hyphen/js/src/HyphenApplication.cpp @@ -0,0 +1 @@ +#include "HyphenApplication.h" \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/src/HyphenApplication.h b/Common/3dParty/hyphen/js/src/HyphenApplication.h new file mode 100644 index 0000000000..69d900e5f8 --- /dev/null +++ b/Common/3dParty/hyphen/js/src/HyphenApplication.h @@ -0,0 +1,25 @@ +#ifndef HYPHEN_APPLICATION_H +#define HYPHEN_APPLICATION_H + +#include +#include +#include + +#include "./../../hyphen/hyphen.h" + +class CHyphenApplication +{ +public: + CHyphenApplication(); + ~CHyphenApplication(); + + char* hyphenWord(const char *word, size_t size, const char *lang); + void loadDictionary(const char *dict, size_t size, const char* lang); + +private: + std::map m_mapDicts; + char* m_pHyphenVector; + +}; + +#endif // HYPHEN_APPLICATION_H \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/src/HyphenUtils.cpp b/Common/3dParty/hyphen/js/src/HyphenUtils.cpp new file mode 100644 index 0000000000..64cb5b079e --- /dev/null +++ b/Common/3dParty/hyphen/js/src/HyphenUtils.cpp @@ -0,0 +1,180 @@ +#include + +#include "../../hyphen/hyphen.c" + +HyphenDict * hnj_hyphen_load_stream (std::istream &in) +{ + HyphenDict *dict[2]; + HashTab *hashtab; + char buf[MAX_CHARS]; + int nextlevel = 0; + int i, j, k; + HashEntry *e; + int state_num = 0; + + /* loading one or two dictionaries (separated by NEXTLEVEL keyword) */ + for (k = 0; k < 2; k++) + { + hashtab = hnj_hash_new(); +#ifdef VERBOSE + global[k] = hashtab; +#endif + hnj_hash_insert (hashtab, "", 0); + dict[k] = (HyphenDict *) hnj_malloc (sizeof(HyphenDict)); + dict[k]->num_states = 1; + dict[k]->states = (HyphenState *) hnj_malloc (sizeof(HyphenState)); + dict[k]->states[0].match = NULL; + dict[k]->states[0].repl = NULL; + dict[k]->states[0].fallback_state = -1; + dict[k]->states[0].num_trans = 0; + dict[k]->states[0].trans = NULL; + dict[k]->nextlevel = NULL; + dict[k]->lhmin = 0; + dict[k]->rhmin = 0; + dict[k]->clhmin = 0; + dict[k]->crhmin = 0; + dict[k]->nohyphen = NULL; + dict[k]->nohyphenl = 0; + + /* read in character set info */ + if (k == 0) + { + for (i = 0; i < MAX_NAME; i++) + dict[k]->cset[i]= 0; + + if (in >> buf) + { + for (i = 0; i < MAX_NAME; i++) + if ((dict[k]->cset[i] == '\r') || (dict[k]->cset[i] == '\n')) + dict[k]->cset[i] = 0; + } + else + { + dict[k]->cset[0] = 0; + } + dict[k]->utf8 = (strcmp(dict[k]->cset, "UTF-8") == 0); + } + else + { + strncpy(dict[k]->cset, dict[0]->cset, sizeof(dict[k]->cset)-1); + dict[k]->cset[sizeof(dict[k]->cset)-1] = '\0'; + dict[k]->utf8 = dict[0]->utf8; + } + + if (k == 0 || nextlevel) + { + while (in >> buf) + { + /* discard lines that don't fit in buffer */ + if (!in.eof() && strchr(buf, '\n') == NULL) + { + int c; + while ((c = in.get()) != '\n' && c != EOF); + + /* issue warning if not a comment */ + if (buf[0] != '%') + { + fprintf(stderr, "Warning: skipping too long pattern (more than %lu chars)\n", sizeof(buf)); + } + continue; + } + + if (strncmp(buf, "NEXTLEVEL", 9) == 0) + { + nextlevel = 1; + break; + } + else if (buf[0] != '%') + { + hnj_hyphen_load_line(buf, dict[k], hashtab); + } + } + } + else if (k == 1) + { + /* default first level: hyphen and ASCII apostrophe */ + if (!dict[0]->utf8) + hnj_hyphen_load_line("NOHYPHEN ',-\n", dict[k], hashtab); + + else + hnj_hyphen_load_line("NOHYPHEN ',\xe2\x80\x93,\xe2\x80\x99,-\n", dict[k], hashtab); + + strncpy(buf, "1-1\n", MAX_CHARS - 1); /* buf rewritten by hnj_hyphen_load here */ + buf[MAX_CHARS-1] = '\0'; + hnj_hyphen_load_line(buf, dict[k], hashtab); /* remove hyphen */ + hnj_hyphen_load_line("1'1\n", dict[k], hashtab); /* ASCII apostrophe */ + + if (dict[0]->utf8) + { + hnj_hyphen_load_line("1\xe2\x80\x93" "1\n", dict[k], hashtab); /* endash */ + hnj_hyphen_load_line("1\xe2\x80\x99" "1\n", dict[k], hashtab); /* apostrophe */ + } + } + + /* Could do unioning of matches here (instead of the preprocessor script). + If we did, the pseudocode would look something like this: + + foreach state in the hash table + foreach i = [1..length(state) - 1] + state to check is substr (state, i) + look it up + if found, and if there is a match, union the match in. + + It's also possible to avoid the quadratic blowup by doing the + search in order of increasing state string sizes - then you + can break the loop after finding the first match. + + This step should be optional in any case - if there is a + preprocessed rule table, it's always faster to use that. + +*/ + + /* put in the fallback states */ + for (i = 0; i < HASH_SIZE; i++) + for (e = hashtab->entries[i]; e; e = e->next) + { + if (*(e->key)) for (j = 1; 1; j++) + { + state_num = hnj_hash_lookup (hashtab, e->key + j); + if (state_num >= 0) + break; + } + /* KBH: FIXME state 0 fallback_state should always be -1? */ + if (e->val) + dict[k]->states[e->val].fallback_state = state_num; + } + +#ifdef VERBOSE + for (i = 0; i < HASH_SIZE; i++) + for (e = hashtab->entries[i]; e; e = e->next) + { + printf ("%d string %s state %d, fallback=%d\n", i, e->key, e->val, + dict[k]->states[e->val].fallback_state); + for (j = 0; j < dict[k]->states[e->val].num_trans; j++) + printf (" %c->%d\n", dict[k]->states[e->val].trans[j].ch, + dict[k]->states[e->val].trans[j].new_state); + } +#endif + +#ifndef VERBOSE + hnj_hash_free (hashtab); +#endif + state_num = 0; + } + if (nextlevel) dict[0]->nextlevel = dict[1]; + else + { + dict[1] -> nextlevel = dict[0]; + dict[1]->lhmin = dict[0]->lhmin; + dict[1]->rhmin = dict[0]->rhmin; + dict[1]->clhmin = (dict[0]->clhmin) ? dict[0]->clhmin : ((dict[0]->lhmin) ? dict[0]->lhmin : 3); + dict[1]->crhmin = (dict[0]->crhmin) ? dict[0]->crhmin : ((dict[0]->rhmin) ? dict[0]->rhmin : 3); +#ifdef VERBOSE + HashTab *r = global[0]; + global[0] = global[1]; + global[1] = r; +#endif + return dict[1]; + } + return dict[0]; +} diff --git a/Common/3dParty/hyphen/js/src/exported_functions.cpp b/Common/3dParty/hyphen/js/src/exported_functions.cpp index 4dd1826c57..a8d4c863a5 100644 --- a/Common/3dParty/hyphen/js/src/exported_functions.cpp +++ b/Common/3dParty/hyphen/js/src/exported_functions.cpp @@ -1,25 +1,42 @@ -#include "exported_functions.h" #include +#include +#include + +#include "exported_functions.h" +#include "HyphenUtils.cpp" + +// CHyphenApplication* createHyphenApplication() +// { +// return new CHyphenApplication(); +// } +// void destoyHyphenApplication(CHyphenApplication *app) +// { +// delete app; +// } HyphenDict *dict; + void load_dictionary() { - dict = hnj_hyphen_load("dictionaries/hyph_ru_RU.dic"); + dict = hnj_hyphen_load("dictionaries/hyph_ru_RU.dic"); + // stringstream ss + // dict = hnj_hyphen_load_stream(fin) } void free_dictionary() { - hnj_hyphen_free(dict); + hnj_hyphen_free(dict); } void hyphenate(const char *word, char *hyphens, char *hword) { - int n = strlen(word); + int n = strlen(word); - char **rep = NULL; - int *pos = NULL; - int *cut = NULL; - - hnj_hyphen_hyphenate2(dict, word, n, hyphens, hword, &rep, &pos, &cut); + char **rep = NULL; + int *pos = NULL; + int *cut = NULL; + + hnj_hyphen_hyphenate2(dict, word, n, hyphens, hword, &rep, &pos, &cut); } -// emcc src/exported_functions.cpp ../hyphen/hyphen.c ../hyphen/hnjalloc.c -o deploy/effects.js -sEXPORTED_FUNCTIONS='_malloc','_free','_hyphenate','_load_dictionary','_free_dictionary' -sEXPORTED_RUNTIME_METHODS='stringToUTF8','UTF8ToString' --preload-file dictionaries \ No newline at end of file +// emcc src/exported_functions.cpp src/HyphenUtils.cpp -o deploy/hyphen.js -sEXPORTED_FUNCTIONS='_malloc','_free','_hyphenate','_load_dictionary','_free_dictionary' -sEXPORTED_RUNTIME_METHODS='stringToUTF8','UTF8ToString' --preload-file dictionaries -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS=0 +// emcc src/exported_functions.cpp src/CHYphenApplication.cpp ../hyphen/hyphen.c ../hyphen/hnjalloc.c -o deploy/hyphen.js -sEXPORTED_FUNCTIONS='_malloc','_free','_hyphenate','_load_dictionary','_free_dictionary' -sEXPORTED_RUNTIME_METHODS='stringToUTF8','UTF8ToString' --preload-file dictionaries -sALLOW_MEMORY_GROWTH \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/src/exported_functions.h b/Common/3dParty/hyphen/js/src/exported_functions.h index 6d5cb0b84d..be2514c628 100644 --- a/Common/3dParty/hyphen/js/src/exported_functions.h +++ b/Common/3dParty/hyphen/js/src/exported_functions.h @@ -1,6 +1,7 @@ -#include +#ifndef EXPORTED_FUNCTIONS_H +#define EXPORTED_FUNCTIONS_H -#include "./../../hyphen/hyphen.h" +#include #ifdef __cplusplus extern "C" @@ -13,4 +14,6 @@ void hyphenate(const char *word, char *hyphens, char *hword); #ifdef __cplusplus } -#endif // __cplusplus \ No newline at end of file +#endif // __cplusplus + +#endif // EXPORTED_FUNCTIONS_H \ No newline at end of file From 50870e499ba1cb88b53d12c7ed4cf64ecddc005e Mon Sep 17 00:00:00 2001 From: Alexey Date: Mon, 19 Sep 2022 16:42:01 +0300 Subject: [PATCH 05/17] JSON for make.py add --- .../hyphen/js/hyphen-build/hyphen.json | 35 +++++++++++++++++++ .../3dParty/hyphen/js/{main.js => module.js} | 6 ++++ 2 files changed, 41 insertions(+) create mode 100644 Common/3dParty/hyphen/js/hyphen-build/hyphen.json rename Common/3dParty/hyphen/js/{main.js => module.js} (98%) diff --git a/Common/3dParty/hyphen/js/hyphen-build/hyphen.json b/Common/3dParty/hyphen/js/hyphen-build/hyphen.json new file mode 100644 index 0000000000..5d87cccc3e --- /dev/null +++ b/Common/3dParty/hyphen/js/hyphen-build/hyphen.json @@ -0,0 +1,35 @@ +{ + "name": "hyphen", + "res_folder": "../deploy", + "wasm": true, + "asm": true, + "embed_mem_file": true, + "run_before": "", + "run_after": "", + "base_js_content": "../module.js", + + "compiler_flags": [ + "-O3", + "-fno-exceptions", + "-fno-rtti", + "-Wno-unused-command-line-argument", + "-sALLOW_MEMORY_GROWTH", + "-sERROR_ON_UNDEFINED_SYMBOLS=0" + ], + "exported_functions": [ + "_malloc", + "_free", + "_load_dictionary", + "_free_dictionary", + "_hyphenate" + ], + "include_path": ["../src"], + "define": [], + "compile_files_array": [ + { + "name": "s", + "folder": "../src", + "files": ["exported_functions.cpp", "HyphenUtils.cpp", "../../hyphen/hyphen.c", "../../hyphen/hnjalloc.c"] + } + ] +} diff --git a/Common/3dParty/hyphen/js/main.js b/Common/3dParty/hyphen/js/module.js similarity index 98% rename from Common/3dParty/hyphen/js/main.js rename to Common/3dParty/hyphen/js/module.js index 1800062f87..c9f8f43f13 100644 --- a/Common/3dParty/hyphen/js/main.js +++ b/Common/3dParty/hyphen/js/module.js @@ -1,3 +1,9 @@ +//desktop_fetch + +// polyfill + +// module + /** * @param {Module} module * @param {Number} length From de4c2884905331380c3741328b42ba1e9a31fad8 Mon Sep 17 00:00:00 2001 From: Alexey Date: Tue, 20 Sep 2022 16:53:58 +0300 Subject: [PATCH 06/17] Hyphen update Add load dictionaries Add HyphenApplication Add load dictionaries from stream --- Common/3dParty/hyphen/.gitignore | 1 + .../hyphen/js/hyphen-build/hyphen.json | 13 +- Common/3dParty/hyphen/js/index.html | 2 +- Common/3dParty/hyphen/js/module.js | 176 ++++++++++++------ .../hyphen/js/src/HyphenApplication.cpp | 60 +++++- .../3dParty/hyphen/js/src/HyphenApplication.h | 8 +- Common/3dParty/hyphen/js/src/HyphenUtils.cpp | 10 +- .../hyphen/js/src/exported_functions.cpp | 43 ++--- .../hyphen/js/src/exported_functions.h | 10 +- 9 files changed, 219 insertions(+), 104 deletions(-) diff --git a/Common/3dParty/hyphen/.gitignore b/Common/3dParty/hyphen/.gitignore index bb01c12494..9c67b974e0 100644 --- a/Common/3dParty/hyphen/.gitignore +++ b/Common/3dParty/hyphen/.gitignore @@ -5,5 +5,6 @@ msvc_make.bat build-test-* *.wasm hyphen.js +hyphen_ie.js hyphen.data .vscode \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/hyphen-build/hyphen.json b/Common/3dParty/hyphen/js/hyphen-build/hyphen.json index 5d87cccc3e..c5d280d862 100644 --- a/Common/3dParty/hyphen/js/hyphen-build/hyphen.json +++ b/Common/3dParty/hyphen/js/hyphen-build/hyphen.json @@ -10,18 +10,19 @@ "compiler_flags": [ "-O3", - "-fno-exceptions", + "-sASSERTIONS", "-fno-rtti", "-Wno-unused-command-line-argument", "-sALLOW_MEMORY_GROWTH", - "-sERROR_ON_UNDEFINED_SYMBOLS=0" + "-sEXPORTED_RUNTIME_METHODS='stringToUTF8','UTF8ToString'" ], "exported_functions": [ "_malloc", "_free", - "_load_dictionary", - "_free_dictionary", - "_hyphenate" + "_createHyphenApplication", + "_destroyHyphenApplication", + "_loadDictionary", + "_hyphenWord" ], "include_path": ["../src"], "define": [], @@ -29,7 +30,7 @@ { "name": "s", "folder": "../src", - "files": ["exported_functions.cpp", "HyphenUtils.cpp", "../../hyphen/hyphen.c", "../../hyphen/hnjalloc.c"] + "files": ["exported_functions.cpp", "HyphenApplication.cpp", "../../hyphen/hnjalloc.c"] } ] } diff --git a/Common/3dParty/hyphen/js/index.html b/Common/3dParty/hyphen/js/index.html index 1b2c1978a0..b2b3725ad6 100644 --- a/Common/3dParty/hyphen/js/index.html +++ b/Common/3dParty/hyphen/js/index.html @@ -12,6 +12,6 @@ - + \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index c9f8f43f13..ea3588ff74 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -1,8 +1,8 @@ //desktop_fetch -// polyfill +//polyfill -// module +//module /** * @param {Module} module @@ -29,73 +29,133 @@ function freeMemory(module, ptr) { * @param {Module} module * @param {Uint8Array} data * @param {Number} ptr - * @return {void} * Fill memory */ function setMemory(module, data, ptr) { module.HEAP8.set(data, ptr); } +/** + * + * @param {Module} module + * @return {Number} + * Returns pointer + */ +function createHyphenApplication(module){ + return module._createHyphenApplication(); +} +/** + * + * @param {Module} module + * @param {Number} app + */ +function destroyHyphenApplication(module, app){ + module._destroyHyphenApplication(app); +} +/** + * + * @param {Module} module + * @param {Number} app + * @param {String} src + * @param {String} lang + */ +function loadDictionary(module, app, src, lang){ + + var src_size = 4 * src.length + 1; + var lang_size = 4 * lang.length + 1; + + + var _src = allocateMemory(module, src_size); + var _lang = allocateMemory(module, lang_size); + + module.stringToUTF8(src, _src, src_size); + module.stringToUTF8(lang, _lang, lang_size); + + module._loadDictionary(app, _src, _lang); + + freeMemory(module, _src); + freeMemory(module, _lang); +} /** - * @type {HTMLTextAreaElement} + * + * @param {Module} module + * @param {Number} app + * @param {String} word + * @param {String} lang + * @returns {Uint8ClampedArray} + * Returns hyphen vector of word */ -var textarea = document.getElementById("textarea"); -var form = document.querySelector("form"); +function hyphenWord(module, app, word, lang){ -var control = document.getElementById("dictionary-file"); -control.addEventListener("change", function(event) { + var word_size = 4 * word.length + 1; + var lang_size = 4 * lang.length + 1; + + var _word = allocateMemory(module, word_size); + var _lang = allocateMemory(module, lang_size); + + module.stringToUTF8(word, _word, word_size); + module.stringToUTF8(lang, _lang, lang_size); + + var hyphens = module._hyphenWord(app, _word, _lang); + + freeMemory(module, _word); + freeMemory(module, _lang); + + return hyphens; +} + +(function (window, undefined){ + var textarea = document.getElementById("textarea"); + var form = document.querySelector("form"); + + var control = document.getElementById("dictionary-file"); + + var application = undefined; + var lang = undefined; + + control.addEventListener("change", function(event) { + var file = control.files[0]; + lang = file.name; + var reader = new FileReader(); + reader.readAsText(file); - var file = control.files[0]; - var reader = new FileReader(); - reader.readAsText(file); - - reader.onload = function() { - - }; - -}, false); - -function hyphenate(word) -{ - var len = word.length; - var word_size = 4 * len + 1; - var hyphen_size = word_size + 5; - var hword_size = word_size * 2; + if(application == undefined) { + application = createHyphenApplication(Module); + } - var pword = allocateMemory(Module, word_size); - var phyphens = allocateMemory(Module, hyphen_size); - var phword = allocateMemory(Module, hword_size); - - Module.stringToUTF8(word, pword, word_size); - Module._hyphenate(pword, phyphens, phword) - - var hword = Module.UTF8ToString(phword); - var hyphens = Module.UTF8ToString(phyphens); - - var positions = []; - for(var i = 0; i < hyphens.length; i++) { - // hyphenation vector has odd number in positions where hyphen is needed - if(Number(hyphens[i]) % 2 == 1) { - positions.push(i); + reader.onload = function() { + loadDictionary(Module, application, reader.result, lang); + }; + + }, false); + + form.onsubmit = function(event) { + event.preventDefault(); + var text = textarea.value.split("\n").join(" ").split(" "); + + if(application == undefined) { + application = createHyphenApplication(Module); + } + + for(var i = 0; i < text.length; i++) { + const ptr = hyphenWord(Module, application, text[i].toLowerCase(), lang); + var vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, 4 * text[i].length + 6); + + // calc actual size of vector + var size = 0; + for(var j = 0; j < vector.length && vector[j] != 0; j++){ + size++; + } + + // size of one symbol + var symbol = size / text[i].length; + + for(var j = 0; j < vector.length; j++) { + if(vector[j] % 2 == 1) { + console.log((j + 1) / symbol); + } + } } } - console.log(hword); - - freeMemory(Module, pword); - freeMemory(Module, phyphens); - freeMemory(Module, phword); - - return positions; -} - -form.onsubmit = function(event) { - event.preventDefault(); - var text = textarea.value.split("\n").join(" ").split(" "); - Module._load_dictionary(); - - for(var i = 0; i < text.length; i++) { - console.log(hyphenate(text[i].toLowerCase())); - } - Module._free_dictionary(); -} +})(self, undefined); diff --git a/Common/3dParty/hyphen/js/src/HyphenApplication.cpp b/Common/3dParty/hyphen/js/src/HyphenApplication.cpp index 32a3c28470..57170ab90e 100644 --- a/Common/3dParty/hyphen/js/src/HyphenApplication.cpp +++ b/Common/3dParty/hyphen/js/src/HyphenApplication.cpp @@ -1 +1,59 @@ -#include "HyphenApplication.h" \ No newline at end of file +#include "HyphenApplication.h" + +#include "HyphenUtils.cpp" + +#define HYPHEN_VECTOR_SIZE 100 +#define MAX_DICTS_COUNT 10 + +CHyphenApplication::CHyphenApplication() : + m_nHyphenVectorSize(HYPHEN_VECTOR_SIZE), m_nMaxDictsCount(MAX_DICTS_COUNT) +{ + m_pHyphenVector = new char[m_nHyphenVectorSize]; +} +CHyphenApplication::~CHyphenApplication() +{ + delete[] m_pHyphenVector; +} + +char* CHyphenApplication::hyphenWord(const char *word, const char *lang) +{ + std::string s_lang(lang); + HyphenDict *dict = m_mapDicts[s_lang]; + size_t n = strlen(word); + + // resize 2x + if(n + 5 > m_nHyphenVectorSize) + { + delete[] m_pHyphenVector; + m_nHyphenVectorSize *= 2; + m_pHyphenVector = new char[m_nHyphenVectorSize]; + } + + memset(m_pHyphenVector, 0, m_nHyphenVectorSize); + + char **rep = NULL; + int *pos = NULL; + int *cut = NULL; + char *hword = new char[n * 2]; + + hnj_hyphen_hyphenate2(dict, word, n, m_pHyphenVector, hword, &rep, &pos, &cut); + + delete[] hword; + return m_pHyphenVector; +} +void CHyphenApplication::loadDictionary(const char *src, const char* lang) +{ + std::string s_lang(lang); + HyphenDict *dict; + + if(m_mapDicts[s_lang] != nullptr) + return; + + if(m_mapDicts.size() > m_nMaxDictsCount) + m_mapDicts.erase(m_mapDicts.begin()); + + std::stringstream ss(src); + dict = hnj_hyphen_load_stream(ss); + + m_mapDicts[s_lang] = dict; +} \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/src/HyphenApplication.h b/Common/3dParty/hyphen/js/src/HyphenApplication.h index 69d900e5f8..f47cd63421 100644 --- a/Common/3dParty/hyphen/js/src/HyphenApplication.h +++ b/Common/3dParty/hyphen/js/src/HyphenApplication.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "./../../hyphen/hyphen.h" @@ -13,12 +14,15 @@ public: CHyphenApplication(); ~CHyphenApplication(); - char* hyphenWord(const char *word, size_t size, const char *lang); - void loadDictionary(const char *dict, size_t size, const char* lang); + char* hyphenWord(const char *word, const char *lang); + void loadDictionary(const char *src, const char *lang); private: std::map m_mapDicts; + size_t m_nMaxDictsCount; + char* m_pHyphenVector; + size_t m_nHyphenVectorSize; }; diff --git a/Common/3dParty/hyphen/js/src/HyphenUtils.cpp b/Common/3dParty/hyphen/js/src/HyphenUtils.cpp index 64cb5b079e..3f8cfb920c 100644 --- a/Common/3dParty/hyphen/js/src/HyphenUtils.cpp +++ b/Common/3dParty/hyphen/js/src/HyphenUtils.cpp @@ -1,6 +1,11 @@ #include +#include +extern "C" +{ #include "../../hyphen/hyphen.c" +#include "../../hyphen/hnjalloc.h" +} HyphenDict * hnj_hyphen_load_stream (std::istream &in) { @@ -63,8 +68,11 @@ HyphenDict * hnj_hyphen_load_stream (std::istream &in) if (k == 0 || nextlevel) { - while (in >> buf) + while (in.getline(buf, sizeof(buf), '\n')) { + if(strlen(buf) < sizeof(buf)) + strcat(buf, "\n"); + /* discard lines that don't fit in buffer */ if (!in.eof() && strchr(buf, '\n') == NULL) { diff --git a/Common/3dParty/hyphen/js/src/exported_functions.cpp b/Common/3dParty/hyphen/js/src/exported_functions.cpp index a8d4c863a5..d0bae65ca4 100644 --- a/Common/3dParty/hyphen/js/src/exported_functions.cpp +++ b/Common/3dParty/hyphen/js/src/exported_functions.cpp @@ -1,42 +1,23 @@ -#include -#include -#include - #include "exported_functions.h" -#include "HyphenUtils.cpp" -// CHyphenApplication* createHyphenApplication() -// { -// return new CHyphenApplication(); -// } -// void destoyHyphenApplication(CHyphenApplication *app) -// { -// delete app; -// } - -HyphenDict *dict; - - -void load_dictionary() +CHyphenApplication* createHyphenApplication() { - dict = hnj_hyphen_load("dictionaries/hyph_ru_RU.dic"); - // stringstream ss - // dict = hnj_hyphen_load_stream(fin) + return new CHyphenApplication(); } -void free_dictionary() +void destroyHyphenApplication(CHyphenApplication *app) { - hnj_hyphen_free(dict); + delete app; } -void hyphenate(const char *word, char *hyphens, char *hword) +void loadDictionary(CHyphenApplication *app, const char *src, const char* lang) { - int n = strlen(word); - - char **rep = NULL; - int *pos = NULL; - int *cut = NULL; - - hnj_hyphen_hyphenate2(dict, word, n, hyphens, hword, &rep, &pos, &cut); + app->loadDictionary(src, lang); } +char* hyphenWord(CHyphenApplication *app, const char *word, const char* lang) +{ + return app->hyphenWord(word, lang); +} + + // emcc src/exported_functions.cpp src/HyphenUtils.cpp -o deploy/hyphen.js -sEXPORTED_FUNCTIONS='_malloc','_free','_hyphenate','_load_dictionary','_free_dictionary' -sEXPORTED_RUNTIME_METHODS='stringToUTF8','UTF8ToString' --preload-file dictionaries -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS=0 // emcc src/exported_functions.cpp src/CHYphenApplication.cpp ../hyphen/hyphen.c ../hyphen/hnjalloc.c -o deploy/hyphen.js -sEXPORTED_FUNCTIONS='_malloc','_free','_hyphenate','_load_dictionary','_free_dictionary' -sEXPORTED_RUNTIME_METHODS='stringToUTF8','UTF8ToString' --preload-file dictionaries -sALLOW_MEMORY_GROWTH \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/src/exported_functions.h b/Common/3dParty/hyphen/js/src/exported_functions.h index be2514c628..985c6f7a75 100644 --- a/Common/3dParty/hyphen/js/src/exported_functions.h +++ b/Common/3dParty/hyphen/js/src/exported_functions.h @@ -1,16 +1,18 @@ #ifndef EXPORTED_FUNCTIONS_H #define EXPORTED_FUNCTIONS_H -#include +#include "HyphenApplication.h" #ifdef __cplusplus extern "C" { #endif // __cplusplus -void load_dictionary(); -void free_dictionary(); -void hyphenate(const char *word, char *hyphens, char *hword); +CHyphenApplication* createHyphenApplication(); +void destroyHyphenApplication(CHyphenApplication *app); + +void loadDictionary(CHyphenApplication *app, const char *src, const char* lang); +char* hyphenWord(CHyphenApplication *app, const char *word, const char* lang); #ifdef __cplusplus } From 9205100eb36119cb286183d46383b711b791455e Mon Sep 17 00:00:00 2001 From: Alexey Date: Wed, 21 Sep 2022 19:28:08 +0300 Subject: [PATCH 07/17] Hyphen update + refactoring --- .../hyphen/{test => hyphen_test}/main.cpp | 0 .../hyphen/{test => hyphen_test}/test.pro | 0 .../hyphen.json | 2 +- Common/3dParty/hyphen/js/module.js | 103 ++++++------------ ...ed_functions.cpp => ExportedFunctions.cpp} | 2 +- ...ported_functions.h => ExportedFunctions.h} | 0 .../hyphen/js/src/HyphenApplication.cpp | 10 +- .../3dParty/hyphen/js/{ => test}/index.html | 4 +- Common/3dParty/hyphen/js/test/main.js | 39 +++++++ .../3dParty/hyphen/js/{ => test}/styles.css | 0 10 files changed, 81 insertions(+), 79 deletions(-) rename Common/3dParty/hyphen/{test => hyphen_test}/main.cpp (100%) rename Common/3dParty/hyphen/{test => hyphen_test}/test.pro (100%) rename Common/3dParty/hyphen/js/{hyphen-build => hyphen_build}/hyphen.json (87%) rename Common/3dParty/hyphen/js/src/{exported_functions.cpp => ExportedFunctions.cpp} (96%) rename Common/3dParty/hyphen/js/src/{exported_functions.h => ExportedFunctions.h} (100%) rename Common/3dParty/hyphen/js/{ => test}/index.html (78%) create mode 100644 Common/3dParty/hyphen/js/test/main.js rename Common/3dParty/hyphen/js/{ => test}/styles.css (100%) diff --git a/Common/3dParty/hyphen/test/main.cpp b/Common/3dParty/hyphen/hyphen_test/main.cpp similarity index 100% rename from Common/3dParty/hyphen/test/main.cpp rename to Common/3dParty/hyphen/hyphen_test/main.cpp diff --git a/Common/3dParty/hyphen/test/test.pro b/Common/3dParty/hyphen/hyphen_test/test.pro similarity index 100% rename from Common/3dParty/hyphen/test/test.pro rename to Common/3dParty/hyphen/hyphen_test/test.pro diff --git a/Common/3dParty/hyphen/js/hyphen-build/hyphen.json b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json similarity index 87% rename from Common/3dParty/hyphen/js/hyphen-build/hyphen.json rename to Common/3dParty/hyphen/js/hyphen_build/hyphen.json index c5d280d862..2e24744bb8 100644 --- a/Common/3dParty/hyphen/js/hyphen-build/hyphen.json +++ b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json @@ -30,7 +30,7 @@ { "name": "s", "folder": "../src", - "files": ["exported_functions.cpp", "HyphenApplication.cpp", "../../hyphen/hnjalloc.c"] + "files": ["ExportedFunctions.cpp", "HyphenApplication.cpp", "../../hyphen/hnjalloc.c"] } ] } diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index ea3588ff74..dae3038a99 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -10,7 +10,7 @@ * @return {Number} * Allocate memory for wasm, returns pointer */ -function allocateMemory(module, length) { +function hyphenAllocateMemory(module, length) { const ptr = module._malloc(length); return ptr; } @@ -21,7 +21,7 @@ function allocateMemory(module, length) { * @return {void} * Free memory */ -function freeMemory(module, ptr) { +function hyphenFreeMemory(module, ptr) { module._free(ptr); } @@ -31,7 +31,7 @@ function freeMemory(module, ptr) { * @param {Number} ptr * Fill memory */ -function setMemory(module, data, ptr) { +function hyphenSetMemory(module, data, ptr) { module.HEAP8.set(data, ptr); } /** @@ -40,7 +40,7 @@ function setMemory(module, data, ptr) { * @return {Number} * Returns pointer */ -function createHyphenApplication(module){ +function hyphenCreateHyphenApplication(module){ return module._createHyphenApplication(); } /** @@ -48,7 +48,7 @@ function createHyphenApplication(module){ * @param {Module} module * @param {Number} app */ -function destroyHyphenApplication(module, app){ +function hyphenDestroyHyphenApplication(module, app){ module._destroyHyphenApplication(app); } /** @@ -58,22 +58,21 @@ function destroyHyphenApplication(module, app){ * @param {String} src * @param {String} lang */ -function loadDictionary(module, app, src, lang){ +function hyphenLoadDictionary(module, app, src, lang){ var src_size = 4 * src.length + 1; var lang_size = 4 * lang.length + 1; - - var _src = allocateMemory(module, src_size); - var _lang = allocateMemory(module, lang_size); + var _src = hyphenAllocateMemory(module, src_size); + var _lang = hyphenAllocateMemory(module, lang_size); module.stringToUTF8(src, _src, src_size); module.stringToUTF8(lang, _lang, lang_size); module._loadDictionary(app, _src, _lang); - freeMemory(module, _src); - freeMemory(module, _lang); + hyphenFreeMemory(module, _src); + hyphenFreeMemory(module, _lang); } /** @@ -90,72 +89,34 @@ function hyphenWord(module, app, word, lang){ var word_size = 4 * word.length + 1; var lang_size = 4 * lang.length + 1; - var _word = allocateMemory(module, word_size); - var _lang = allocateMemory(module, lang_size); + var _word = hyphenAllocateMemory(module, word_size); + var _lang = hyphenAllocateMemory(module, lang_size); module.stringToUTF8(word, _word, word_size); module.stringToUTF8(lang, _lang, lang_size); - var hyphens = module._hyphenWord(app, _word, _lang); + const ptr = module._hyphenWord(app, _word, _lang); - freeMemory(module, _word); - freeMemory(module, _lang); + hyphenFreeMemory(module, _word); + hyphenFreeMemory(module, _lang); + var hyphens = []; + var vector = new Uint8ClampedArray(module.HEAP8.buffer, ptr, 4 * word.length + 6); + + // calc actual size of vector + var size = 0; + for(var i = 0; i < vector.length && vector[i] != 0; i++) { + size++; + } + + // size of one symbol + var symbol = size / word.length; + + for(var i = 0; i < vector.length; i++) { + if(vector[i] % 2 == 1) { + hyphens.push((i + 1) / symbol); + } + } return hyphens; } -(function (window, undefined){ - var textarea = document.getElementById("textarea"); - var form = document.querySelector("form"); - - var control = document.getElementById("dictionary-file"); - - var application = undefined; - var lang = undefined; - - control.addEventListener("change", function(event) { - var file = control.files[0]; - lang = file.name; - var reader = new FileReader(); - reader.readAsText(file); - - if(application == undefined) { - application = createHyphenApplication(Module); - } - - reader.onload = function() { - loadDictionary(Module, application, reader.result, lang); - }; - - }, false); - - form.onsubmit = function(event) { - event.preventDefault(); - var text = textarea.value.split("\n").join(" ").split(" "); - - if(application == undefined) { - application = createHyphenApplication(Module); - } - - for(var i = 0; i < text.length; i++) { - const ptr = hyphenWord(Module, application, text[i].toLowerCase(), lang); - var vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, 4 * text[i].length + 6); - - // calc actual size of vector - var size = 0; - for(var j = 0; j < vector.length && vector[j] != 0; j++){ - size++; - } - - // size of one symbol - var symbol = size / text[i].length; - - for(var j = 0; j < vector.length; j++) { - if(vector[j] % 2 == 1) { - console.log((j + 1) / symbol); - } - } - } - } -})(self, undefined); - diff --git a/Common/3dParty/hyphen/js/src/exported_functions.cpp b/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp similarity index 96% rename from Common/3dParty/hyphen/js/src/exported_functions.cpp rename to Common/3dParty/hyphen/js/src/ExportedFunctions.cpp index d0bae65ca4..50f9417f23 100644 --- a/Common/3dParty/hyphen/js/src/exported_functions.cpp +++ b/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp @@ -1,4 +1,4 @@ -#include "exported_functions.h" +#include "ExportedFunctions.h" CHyphenApplication* createHyphenApplication() { diff --git a/Common/3dParty/hyphen/js/src/exported_functions.h b/Common/3dParty/hyphen/js/src/ExportedFunctions.h similarity index 100% rename from Common/3dParty/hyphen/js/src/exported_functions.h rename to Common/3dParty/hyphen/js/src/ExportedFunctions.h diff --git a/Common/3dParty/hyphen/js/src/HyphenApplication.cpp b/Common/3dParty/hyphen/js/src/HyphenApplication.cpp index 57170ab90e..baf7c43d0b 100644 --- a/Common/3dParty/hyphen/js/src/HyphenApplication.cpp +++ b/Common/3dParty/hyphen/js/src/HyphenApplication.cpp @@ -34,11 +34,9 @@ char* CHyphenApplication::hyphenWord(const char *word, const char *lang) char **rep = NULL; int *pos = NULL; int *cut = NULL; - char *hword = new char[n * 2]; - hnj_hyphen_hyphenate2(dict, word, n, m_pHyphenVector, hword, &rep, &pos, &cut); + hnj_hyphen_hyphenate2(dict, word, n, m_pHyphenVector, NULL, &rep, &pos, &cut); - delete[] hword; return m_pHyphenVector; } void CHyphenApplication::loadDictionary(const char *src, const char* lang) @@ -50,7 +48,11 @@ void CHyphenApplication::loadDictionary(const char *src, const char* lang) return; if(m_mapDicts.size() > m_nMaxDictsCount) - m_mapDicts.erase(m_mapDicts.begin()); + { + auto it = m_mapDicts.begin(); + hnj_hyphen_free(it->second); + m_mapDicts.erase(it); + } std::stringstream ss(src); dict = hnj_hyphen_load_stream(ss); diff --git a/Common/3dParty/hyphen/js/index.html b/Common/3dParty/hyphen/js/test/index.html similarity index 78% rename from Common/3dParty/hyphen/js/index.html rename to Common/3dParty/hyphen/js/test/index.html index b2b3725ad6..433082e079 100644 --- a/Common/3dParty/hyphen/js/index.html +++ b/Common/3dParty/hyphen/js/test/index.html @@ -11,7 +11,7 @@ - - + + \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/test/main.js b/Common/3dParty/hyphen/js/test/main.js new file mode 100644 index 0000000000..6f36c052a5 --- /dev/null +++ b/Common/3dParty/hyphen/js/test/main.js @@ -0,0 +1,39 @@ +(function (window, undefined){ + var textarea = document.getElementById("textarea"); + var form = document.querySelector("form"); + + var control = document.getElementById("dictionary-file"); + + var application = undefined; + var lang = undefined; + + control.addEventListener("change", function(event) { + var file = control.files[0]; + lang = file.name; + var reader = new FileReader(); + reader.readAsText(file); + + if(application == undefined) { + application = hyphenCreateHyphenApplication(Module); + } + + reader.onload = function() { + hyphenLoadDictionary(Module, application, reader.result, lang); + }; + + }, false); + + form.onsubmit = function(event) { + event.preventDefault(); + var text = textarea.value.split("\n").join(" ").split(" "); + + if(application == undefined) { + application = hyphenCreateHyphenApplication(Module); + } + + for(var i = 0; i < text.length; i++) { + var hyphens = hyphenWord(Module, application, text[i].toLowerCase(), lang); + console.log(hyphens); + } + } +})(self, undefined); \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/styles.css b/Common/3dParty/hyphen/js/test/styles.css similarity index 100% rename from Common/3dParty/hyphen/js/styles.css rename to Common/3dParty/hyphen/js/test/styles.css From 82b5b08dfc48a5da9914a43520ac9e09eb17b4d2 Mon Sep 17 00:00:00 2001 From: Alexey Date: Wed, 21 Sep 2022 19:43:27 +0300 Subject: [PATCH 08/17] Hyphen refactoring --- Common/3dParty/hyphen/js/hyphen_build/hyphen.json | 6 +++--- Common/3dParty/hyphen/js/module.js | 10 +++++----- Common/3dParty/hyphen/js/src/ExportedFunctions.cpp | 6 +++--- Common/3dParty/hyphen/js/src/ExportedFunctions.h | 6 +++--- Common/3dParty/hyphen/js/test/main.js | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json index 2e24744bb8..e07d1b9b69 100644 --- a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json +++ b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json @@ -19,9 +19,9 @@ "exported_functions": [ "_malloc", "_free", - "_createHyphenApplication", - "_destroyHyphenApplication", - "_loadDictionary", + "_hyphenCreateApplication", + "_hyphenDestroyApplication", + "_hyphenLoadDictionary", "_hyphenWord" ], "include_path": ["../src"], diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index dae3038a99..c1ba4e5d76 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -40,16 +40,16 @@ function hyphenSetMemory(module, data, ptr) { * @return {Number} * Returns pointer */ -function hyphenCreateHyphenApplication(module){ - return module._createHyphenApplication(); +function hyphenCreateApplication(module){ + return module._hyphenCreateApplication(); } /** * * @param {Module} module * @param {Number} app */ -function hyphenDestroyHyphenApplication(module, app){ - module._destroyHyphenApplication(app); +function hyphenDestroyApplication(module, app){ + module._hyphenDestroyApplication(app); } /** * @@ -69,7 +69,7 @@ function hyphenLoadDictionary(module, app, src, lang){ module.stringToUTF8(src, _src, src_size); module.stringToUTF8(lang, _lang, lang_size); - module._loadDictionary(app, _src, _lang); + module._hyphenLoadDictionary(app, _src, _lang); hyphenFreeMemory(module, _src); hyphenFreeMemory(module, _lang); diff --git a/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp b/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp index 50f9417f23..01d345f228 100644 --- a/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp +++ b/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp @@ -1,15 +1,15 @@ #include "ExportedFunctions.h" -CHyphenApplication* createHyphenApplication() +CHyphenApplication* hyphenCreateApplication() { return new CHyphenApplication(); } -void destroyHyphenApplication(CHyphenApplication *app) +void hyphenDestroyApplication(CHyphenApplication *app) { delete app; } -void loadDictionary(CHyphenApplication *app, const char *src, const char* lang) +void hyphenLoadDictionary(CHyphenApplication *app, const char *src, const char* lang) { app->loadDictionary(src, lang); } diff --git a/Common/3dParty/hyphen/js/src/ExportedFunctions.h b/Common/3dParty/hyphen/js/src/ExportedFunctions.h index 985c6f7a75..d0764b9864 100644 --- a/Common/3dParty/hyphen/js/src/ExportedFunctions.h +++ b/Common/3dParty/hyphen/js/src/ExportedFunctions.h @@ -8,10 +8,10 @@ extern "C" { #endif // __cplusplus -CHyphenApplication* createHyphenApplication(); -void destroyHyphenApplication(CHyphenApplication *app); +CHyphenApplication* hyphenCreateApplication(); +void hyphenDestroyApplication(CHyphenApplication *app); -void loadDictionary(CHyphenApplication *app, const char *src, const char* lang); +void hyphenLoadDictionary(CHyphenApplication *app, const char *src, const char* lang); char* hyphenWord(CHyphenApplication *app, const char *word, const char* lang); #ifdef __cplusplus diff --git a/Common/3dParty/hyphen/js/test/main.js b/Common/3dParty/hyphen/js/test/main.js index 6f36c052a5..fd7e6e19e5 100644 --- a/Common/3dParty/hyphen/js/test/main.js +++ b/Common/3dParty/hyphen/js/test/main.js @@ -14,7 +14,7 @@ reader.readAsText(file); if(application == undefined) { - application = hyphenCreateHyphenApplication(Module); + application = hyphenCreateApplication(Module); } reader.onload = function() { @@ -28,7 +28,7 @@ var text = textarea.value.split("\n").join(" ").split(" "); if(application == undefined) { - application = hyphenCreateHyphenApplication(Module); + application = hyphenCreateApplication(Module); } for(var i = 0; i < text.length; i++) { From 35cbbe2fb1d936e8e789c3a00c9d0c6f2abd38f6 Mon Sep 17 00:00:00 2001 From: Alexey Date: Thu, 22 Sep 2022 21:53:34 +0300 Subject: [PATCH 09/17] Add combobox in js test --- Common/3dParty/hyphen/js/test/index.html | 37 ++++++++++++++++- Common/3dParty/hyphen/js/test/main.js | 51 ++++++++++++------------ 2 files changed, 61 insertions(+), 27 deletions(-) diff --git a/Common/3dParty/hyphen/js/test/index.html b/Common/3dParty/hyphen/js/test/index.html index 433082e079..067f40bcae 100644 --- a/Common/3dParty/hyphen/js/test/index.html +++ b/Common/3dParty/hyphen/js/test/index.html @@ -9,7 +9,42 @@
- +
diff --git a/Common/3dParty/hyphen/js/test/main.js b/Common/3dParty/hyphen/js/test/main.js index fd7e6e19e5..ddf84621c9 100644 --- a/Common/3dParty/hyphen/js/test/main.js +++ b/Common/3dParty/hyphen/js/test/main.js @@ -1,39 +1,38 @@ (function (window, undefined){ + var textarea = document.getElementById("textarea"); var form = document.querySelector("form"); - - var control = document.getElementById("dictionary-file"); + var combobox = document.getElementById("combobox"); var application = undefined; - var lang = undefined; - - control.addEventListener("change", function(event) { - var file = control.files[0]; - lang = file.name; - var reader = new FileReader(); - reader.readAsText(file); - - if(application == undefined) { - application = hyphenCreateApplication(Module); - } - - reader.onload = function() { - hyphenLoadDictionary(Module, application, reader.result, lang); - }; - - }, false); form.onsubmit = function(event) { - event.preventDefault(); - var text = textarea.value.split("\n").join(" ").split(" "); - + + if(combobox.value == "") { + return; + } + if(application == undefined) { application = hyphenCreateApplication(Module); } - - for(var i = 0; i < text.length; i++) { - var hyphens = hyphenWord(Module, application, text[i].toLowerCase(), lang); - console.log(hyphens); + + var lang = combobox.value; + var text = textarea.value.split("\n").join(" ").split(" "); + + var request = new XMLHttpRequest(); + var path = '../../../../../../dictionaries/' + lang + '/' + 'hyph_' + lang + '.dic'; + request.open('GET', path, true); + request.send(null); + + request.onload = function () { + var dict = request.responseText; + hyphenLoadDictionary(Module, application, dict, lang); + + for(var i = 0; i < text.length; i++) { + var hyphens = hyphenWord(Module, application, text[i].toLowerCase(), lang); + console.log(hyphens); + } } + event.preventDefault(); } })(self, undefined); \ No newline at end of file From f94e98e09da164867cd50e7d91eef0677ef8409d Mon Sep 17 00:00:00 2001 From: Alexey Date: Fri, 23 Sep 2022 22:12:09 +0300 Subject: [PATCH 10/17] Hyphen update --- .../3dParty/hyphen/js/hyphen_build/after.py | 9 + .../hyphen/js/hyphen_build/hyphen.json | 4 +- Common/3dParty/hyphen/js/library.js | 45 +++ Common/3dParty/hyphen/js/module.js | 257 ++++++++++-------- Common/3dParty/hyphen/js/test/main.js | 14 +- 5 files changed, 203 insertions(+), 126 deletions(-) create mode 100644 Common/3dParty/hyphen/js/hyphen_build/after.py create mode 100644 Common/3dParty/hyphen/js/library.js diff --git a/Common/3dParty/hyphen/js/hyphen_build/after.py b/Common/3dParty/hyphen/js/hyphen_build/after.py new file mode 100644 index 0000000000..ba5e473953 --- /dev/null +++ b/Common/3dParty/hyphen/js/hyphen_build/after.py @@ -0,0 +1,9 @@ +import sys +sys.path.append("../../../../../../build_tools/scripts") +import base + +base.configure_common_apps() +base.replaceInFile("../deploy/engine/hyphen.js", "__ATPOSTRUN__=[];", "__ATPOSTRUN__=[onLoadModule];") +base.replaceInFile("../deploy/engine/hyphen_ie.js", "__ATPOSTRUN__=[];", "__ATPOSTRUN__=[onLoadModule];") + +base.copy_file("../library.js", "../deploy/hyphen.js") \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json index e07d1b9b69..da8c1cb6e3 100644 --- a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json +++ b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json @@ -1,11 +1,11 @@ { "name": "hyphen", - "res_folder": "../deploy", + "res_folder": "../deploy/engine", "wasm": true, "asm": true, "embed_mem_file": true, "run_before": "", - "run_after": "", + "run_after": "after.py", "base_js_content": "../module.js", "compiler_flags": [ diff --git a/Common/3dParty/hyphen/js/library.js b/Common/3dParty/hyphen/js/library.js new file mode 100644 index 0000000000..297cfc1d85 --- /dev/null +++ b/Common/3dParty/hyphen/js/library.js @@ -0,0 +1,45 @@ +(function(window, undefined) { + + window.hyphen = window.hyphen || {}; + window.hyphen.isReady = false; + + var not_ready = function() { + console.log('Module is not ready'); + } + + window.hyphen.hyphenCreateApplication = not_ready; + window.hyphen.hyphenDestroyApplication = not_ready; + window.hyphen.hyphenLoadDictionary = not_ready; + window.hyphen.hyphenWord = not_ready; + + window.hyphen.onLoadModule = function(exports) { + window.hyphen.isReady = true; + + window.hyphen.hyphenCreateApplication = exports.hyphenCreateApplication; + window.hyphen.hyphenDestroyApplication = exports.hyphenDestroyApplication; + window.hyphen.hyphenLoadDictionary = exports.hyphenLoadDictionary; + window.hyphen.hyphenWord = exports.hyphenWord; + }; + + window.hyphen.loadModule = function() { + var path = '../deploy/engine/'; + + // wasm support check + var useWasm = false; + const webAsmObj = window['WebAssembly']; + if (typeof webAsmObj === 'object') { + if (typeof webAsmObj['Memory'] === 'function') { + if ((typeof webAsmObj['instantiateStreaming'] === 'function') || (typeof webAsmObj['instantiate'] === 'function')) { + useWasm = true; + } + } + } + path += (useWasm ? 'hyphen.js' : 'hyphen_ie.js'); + + const script = document.createElement('script'); + script.type = 'text/javascript'; + script.src = path; + document.head.appendChild(script); + + } +})(self); \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index c1ba4e5d76..4ec8fc6048 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -1,122 +1,143 @@ -//desktop_fetch +(function(window, undefined) { -//polyfill + var isModuleLoaded = false; -//module - -/** - * @param {Module} module - * @param {Number} length - * @return {Number} - * Allocate memory for wasm, returns pointer - */ -function hyphenAllocateMemory(module, length) { - const ptr = module._malloc(length); - return ptr; -} - -/** - * @param {Module} module - * @param {Number} ptr - * @return {void} - * Free memory - */ -function hyphenFreeMemory(module, ptr) { - module._free(ptr); -} - -/** - * @param {Module} module - * @param {Uint8Array} data - * @param {Number} ptr - * Fill memory - */ -function hyphenSetMemory(module, data, ptr) { - module.HEAP8.set(data, ptr); -} -/** - * - * @param {Module} module - * @return {Number} - * Returns pointer - */ -function hyphenCreateApplication(module){ - return module._hyphenCreateApplication(); -} -/** - * - * @param {Module} module - * @param {Number} app - */ -function hyphenDestroyApplication(module, app){ - module._hyphenDestroyApplication(app); -} -/** - * - * @param {Module} module - * @param {Number} app - * @param {String} src - * @param {String} lang - */ -function hyphenLoadDictionary(module, app, src, lang){ - - var src_size = 4 * src.length + 1; - var lang_size = 4 * lang.length + 1; - - var _src = hyphenAllocateMemory(module, src_size); - var _lang = hyphenAllocateMemory(module, lang_size); - - module.stringToUTF8(src, _src, src_size); - module.stringToUTF8(lang, _lang, lang_size); - - module._hyphenLoadDictionary(app, _src, _lang); - - hyphenFreeMemory(module, _src); - hyphenFreeMemory(module, _lang); -} - -/** - * - * @param {Module} module - * @param {Number} app - * @param {String} word - * @param {String} lang - * @returns {Uint8ClampedArray} - * Returns hyphen vector of word - */ -function hyphenWord(module, app, word, lang){ - - var word_size = 4 * word.length + 1; - var lang_size = 4 * lang.length + 1; - - var _word = hyphenAllocateMemory(module, word_size); - var _lang = hyphenAllocateMemory(module, lang_size); - - module.stringToUTF8(word, _word, word_size); - module.stringToUTF8(lang, _lang, lang_size); - - const ptr = module._hyphenWord(app, _word, _lang); - - hyphenFreeMemory(module, _word); - hyphenFreeMemory(module, _lang); - - var hyphens = []; - var vector = new Uint8ClampedArray(module.HEAP8.buffer, ptr, 4 * word.length + 6); - - // calc actual size of vector - var size = 0; - for(var i = 0; i < vector.length && vector[i] != 0; i++) { - size++; - } - - // size of one symbol - var symbol = size / word.length; - - for(var i = 0; i < vector.length; i++) { - if(vector[i] % 2 == 1) { - hyphens.push((i + 1) / symbol); + function onLoadModule() { + isModuleLoaded = true; + if (window.hyphen) { + window.hyphen.onLoadModule && window.hyphen.onLoadModule({ + hyphenCreateApplication: hyphenCreateApplication, + hyphenDestroyApplication: hyphenDestroyApplication, + hyphenLoadDictionary: hyphenLoadDictionary, + hyphenWord: hyphenWord, + }); } - } - return hyphens; -} + }; + //desktop_fetch + + //polyfill + + //module + + /** + * @param {Number} length + * @return {Number} + * Allocate memory for wasm, returns pointer + */ + function hyphenAllocateMemory(length) { + const ptr = Module._malloc(length); + return ptr; + } + + /** + * @param {Number} ptr + * @return {void} + * Free memory + */ + function hyphenFreeMemory(ptr) { + Module._free(ptr); + } + + /** + * @param {Uint8Array} data + * @param {Number} ptr + * Fill memory + */ + function hyphenSetMemory(data, ptr) { + Module.HEAP8.set(data, ptr); + } + /** + * + * @return {Number} + * Returns pointer + */ + function hyphenCreateApplication() { + if(!isModuleLoaded) { + return; + } + return Module._hyphenCreateApplication(); + } + /** + * + * @param {Number} app + */ + function hyphenDestroyApplication(app) { + if(!isModuleLoaded) { + return; + } + Module._hyphenDestroyApplication(app); + } + /** + * + * @param {Number} app + * @param {String} src + * @param {String} lang + */ + function hyphenLoadDictionary(app, src, lang) { + if(!isModuleLoaded) { + return; + } + + var src_size = 4 * src.length + 1; + var lang_size = 4 * lang.length + 1; + + var _src = hyphenAllocateMemory(src_size); + var _lang = hyphenAllocateMemory(lang_size); + + Module.stringToUTF8(src, _src, src_size); + Module.stringToUTF8(lang, _lang, lang_size); + + Module._hyphenLoadDictionary(app, _src, _lang); + + hyphenFreeMemory(_src); + hyphenFreeMemory(_lang); + } + + /** + * + * @param {Number} app + * @param {String} word + * @param {String} lang + * @returns {Uint8ClampedArray} + * Returns hyphen vector of word + */ + function hyphenWord(app, word, lang){ + if(!isModuleLoaded) { + return; + } + + var word_size = 4 * word.length + 1; + var lang_size = 4 * lang.length + 1; + + var _word = hyphenAllocateMemory(word_size); + var _lang = hyphenAllocateMemory(lang_size); + + Module.stringToUTF8(word, _word, word_size); + Module.stringToUTF8(lang, _lang, lang_size); + + const ptr = Module._hyphenWord(app, _word, _lang); + + hyphenFreeMemory( _word); + hyphenFreeMemory(_lang); + + var hyphens = []; + var vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, 4 * word.length + 6); + + // calc actual size of vector + var size = 0; + for(var i = 0; i < vector.length && vector[i] != 0; i++) { + size++; + } + + // size of one symbol + var symbol = size / word.length; + + for(var i = 0; i < vector.length; i++) { + if(vector[i] % 2 == 1) { + hyphens.push((i + 1) / symbol); + } + } + return hyphens; + } +})(self); \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/test/main.js b/Common/3dParty/hyphen/js/test/main.js index ddf84621c9..c58ddb789e 100644 --- a/Common/3dParty/hyphen/js/test/main.js +++ b/Common/3dParty/hyphen/js/test/main.js @@ -1,4 +1,6 @@ -(function (window, undefined){ +(function (window, undefined) { + + window.hyphen.loadModule(); var textarea = document.getElementById("textarea"); var form = document.querySelector("form"); @@ -13,7 +15,7 @@ } if(application == undefined) { - application = hyphenCreateApplication(Module); + application = window.hyphen.hyphenCreateApplication(); } var lang = combobox.value; @@ -26,13 +28,13 @@ request.onload = function () { var dict = request.responseText; - hyphenLoadDictionary(Module, application, dict, lang); + window.hyphen.hyphenLoadDictionary(application, dict, lang); for(var i = 0; i < text.length; i++) { - var hyphens = hyphenWord(Module, application, text[i].toLowerCase(), lang); + var hyphens = window.hyphen.hyphenWord(application, text[i].toLowerCase(), lang); console.log(hyphens); } } - event.preventDefault(); + event.preventDefault(); } -})(self, undefined); \ No newline at end of file +})(self); \ No newline at end of file From c591ed20cc078cebd2338618e3da34c5393a6cc1 Mon Sep 17 00:00:00 2001 From: Alexey Date: Fri, 23 Sep 2022 22:18:42 +0300 Subject: [PATCH 11/17] Indents refactoring --- .../hyphen/js/hyphen_build/hyphen.json | 4 +- Common/3dParty/hyphen/js/library.js | 32 +-- Common/3dParty/hyphen/js/module.js | 238 +++++++++--------- Common/3dParty/hyphen/js/test/index.html | 98 ++++---- Common/3dParty/hyphen/js/test/main.js | 60 ++--- Common/3dParty/hyphen/js/test/styles.css | 10 +- 6 files changed, 221 insertions(+), 221 deletions(-) diff --git a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json index da8c1cb6e3..a5d2be5d71 100644 --- a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json +++ b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json @@ -19,8 +19,8 @@ "exported_functions": [ "_malloc", "_free", - "_hyphenCreateApplication", - "_hyphenDestroyApplication", + "_hyphenCreateApplication", + "_hyphenDestroyApplication", "_hyphenLoadDictionary", "_hyphenWord" ], diff --git a/Common/3dParty/hyphen/js/library.js b/Common/3dParty/hyphen/js/library.js index 297cfc1d85..2dfe3b89b1 100644 --- a/Common/3dParty/hyphen/js/library.js +++ b/Common/3dParty/hyphen/js/library.js @@ -1,39 +1,39 @@ (function(window, undefined) { - window.hyphen = window.hyphen || {}; - window.hyphen.isReady = false; + window.hyphen = window.hyphen || {}; + window.hyphen.isReady = false; var not_ready = function() { - console.log('Module is not ready'); + console.log('Module is not ready'); } - window.hyphen.hyphenCreateApplication = not_ready; + window.hyphen.hyphenCreateApplication = not_ready; window.hyphen.hyphenDestroyApplication = not_ready; window.hyphen.hyphenLoadDictionary = not_ready; window.hyphen.hyphenWord = not_ready; - window.hyphen.onLoadModule = function(exports) { - window.hyphen.isReady = true; + window.hyphen.onLoadModule = function(exports) { + window.hyphen.isReady = true; - window.hyphen.hyphenCreateApplication = exports.hyphenCreateApplication; + window.hyphen.hyphenCreateApplication = exports.hyphenCreateApplication; window.hyphen.hyphenDestroyApplication = exports.hyphenDestroyApplication; window.hyphen.hyphenLoadDictionary = exports.hyphenLoadDictionary; window.hyphen.hyphenWord = exports.hyphenWord; - }; + }; window.hyphen.loadModule = function() { var path = '../deploy/engine/'; // wasm support check var useWasm = false; - const webAsmObj = window['WebAssembly']; - if (typeof webAsmObj === 'object') { - if (typeof webAsmObj['Memory'] === 'function') { - if ((typeof webAsmObj['instantiateStreaming'] === 'function') || (typeof webAsmObj['instantiate'] === 'function')) { - useWasm = true; - } - } - } + const webAsmObj = window['WebAssembly']; + if (typeof webAsmObj === 'object') { + if (typeof webAsmObj['Memory'] === 'function') { + if ((typeof webAsmObj['instantiateStreaming'] === 'function') || (typeof webAsmObj['instantiate'] === 'function')) { + useWasm = true; + } + } + } path += (useWasm ? 'hyphen.js' : 'hyphen_ie.js'); const script = document.createElement('script'); diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index 4ec8fc6048..b2a38470ab 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -1,143 +1,143 @@ (function(window, undefined) { - var isModuleLoaded = false; + var isModuleLoaded = false; - function onLoadModule() { - isModuleLoaded = true; - if (window.hyphen) { - window.hyphen.onLoadModule && window.hyphen.onLoadModule({ - hyphenCreateApplication: hyphenCreateApplication, - hyphenDestroyApplication: hyphenDestroyApplication, - hyphenLoadDictionary: hyphenLoadDictionary, - hyphenWord: hyphenWord, - }); - } - }; + function onLoadModule() { + isModuleLoaded = true; + if (window.hyphen) { + window.hyphen.onLoadModule && window.hyphen.onLoadModule({ + hyphenCreateApplication: hyphenCreateApplication, + hyphenDestroyApplication: hyphenDestroyApplication, + hyphenLoadDictionary: hyphenLoadDictionary, + hyphenWord: hyphenWord, + }); + } + }; - //desktop_fetch + //desktop_fetch - //polyfill + //polyfill - //module + //module - /** - * @param {Number} length - * @return {Number} - * Allocate memory for wasm, returns pointer - */ - function hyphenAllocateMemory(length) { - const ptr = Module._malloc(length); - return ptr; - } + /** + * @param {Number} length + * @return {Number} + * Allocate memory for wasm, returns pointer + */ + function hyphenAllocateMemory(length) { + const ptr = Module._malloc(length); + return ptr; + } - /** - * @param {Number} ptr - * @return {void} - * Free memory - */ - function hyphenFreeMemory(ptr) { - Module._free(ptr); - } + /** + * @param {Number} ptr + * @return {void} + * Free memory + */ + function hyphenFreeMemory(ptr) { + Module._free(ptr); + } - /** - * @param {Uint8Array} data - * @param {Number} ptr - * Fill memory - */ - function hyphenSetMemory(data, ptr) { - Module.HEAP8.set(data, ptr); - } - /** - * - * @return {Number} - * Returns pointer - */ - function hyphenCreateApplication() { - if(!isModuleLoaded) { - return; - } - return Module._hyphenCreateApplication(); - } - /** - * - * @param {Number} app - */ - function hyphenDestroyApplication(app) { - if(!isModuleLoaded) { - return; - } - Module._hyphenDestroyApplication(app); - } - /** - * - * @param {Number} app - * @param {String} src - * @param {String} lang - */ - function hyphenLoadDictionary(app, src, lang) { - if(!isModuleLoaded) { - return; - } + /** + * @param {Uint8Array} data + * @param {Number} ptr + * Fill memory + */ + function hyphenSetMemory(data, ptr) { + Module.HEAP8.set(data, ptr); + } + /** + * + * @return {Number} + * Returns pointer + */ + function hyphenCreateApplication() { + if(!isModuleLoaded) { + return; + } + return Module._hyphenCreateApplication(); + } + /** + * + * @param {Number} app + */ + function hyphenDestroyApplication(app) { + if(!isModuleLoaded) { + return; + } + Module._hyphenDestroyApplication(app); + } + /** + * + * @param {Number} app + * @param {String} src + * @param {String} lang + */ + function hyphenLoadDictionary(app, src, lang) { + if(!isModuleLoaded) { + return; + } - var src_size = 4 * src.length + 1; - var lang_size = 4 * lang.length + 1; - - var _src = hyphenAllocateMemory(src_size); - var _lang = hyphenAllocateMemory(lang_size); + var src_size = 4 * src.length + 1; + var lang_size = 4 * lang.length + 1; + + var _src = hyphenAllocateMemory(src_size); + var _lang = hyphenAllocateMemory(lang_size); - Module.stringToUTF8(src, _src, src_size); - Module.stringToUTF8(lang, _lang, lang_size); + Module.stringToUTF8(src, _src, src_size); + Module.stringToUTF8(lang, _lang, lang_size); - Module._hyphenLoadDictionary(app, _src, _lang); + Module._hyphenLoadDictionary(app, _src, _lang); - hyphenFreeMemory(_src); - hyphenFreeMemory(_lang); - } + hyphenFreeMemory(_src); + hyphenFreeMemory(_lang); + } - /** - * - * @param {Number} app - * @param {String} word - * @param {String} lang - * @returns {Uint8ClampedArray} - * Returns hyphen vector of word - */ - function hyphenWord(app, word, lang){ - if(!isModuleLoaded) { - return; - } + /** + * + * @param {Number} app + * @param {String} word + * @param {String} lang + * @returns {Uint8ClampedArray} + * Returns hyphen vector of word + */ + function hyphenWord(app, word, lang){ + if(!isModuleLoaded) { + return; + } - var word_size = 4 * word.length + 1; - var lang_size = 4 * lang.length + 1; + var word_size = 4 * word.length + 1; + var lang_size = 4 * lang.length + 1; - var _word = hyphenAllocateMemory(word_size); - var _lang = hyphenAllocateMemory(lang_size); + var _word = hyphenAllocateMemory(word_size); + var _lang = hyphenAllocateMemory(lang_size); - Module.stringToUTF8(word, _word, word_size); - Module.stringToUTF8(lang, _lang, lang_size); + Module.stringToUTF8(word, _word, word_size); + Module.stringToUTF8(lang, _lang, lang_size); - const ptr = Module._hyphenWord(app, _word, _lang); + const ptr = Module._hyphenWord(app, _word, _lang); - hyphenFreeMemory( _word); - hyphenFreeMemory(_lang); + hyphenFreeMemory( _word); + hyphenFreeMemory(_lang); - var hyphens = []; - var vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, 4 * word.length + 6); + var hyphens = []; + var vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, 4 * word.length + 6); - // calc actual size of vector - var size = 0; - for(var i = 0; i < vector.length && vector[i] != 0; i++) { - size++; - } + // calc actual size of vector + var size = 0; + for(var i = 0; i < vector.length && vector[i] != 0; i++) { + size++; + } - // size of one symbol - var symbol = size / word.length; + // size of one symbol + var symbol = size / word.length; - for(var i = 0; i < vector.length; i++) { - if(vector[i] % 2 == 1) { - hyphens.push((i + 1) / symbol); - } - } - return hyphens; - } + for(var i = 0; i < vector.length; i++) { + if(vector[i] % 2 == 1) { + hyphens.push((i + 1) / symbol); + } + } + return hyphens; + } })(self); \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/test/index.html b/Common/3dParty/hyphen/js/test/index.html index 067f40bcae..77d8bc8601 100644 --- a/Common/3dParty/hyphen/js/test/index.html +++ b/Common/3dParty/hyphen/js/test/index.html @@ -1,52 +1,52 @@ - - - test - - - -
- - - -
- - - + + + test + + + +
+ + + +
+ + + \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/test/main.js b/Common/3dParty/hyphen/js/test/main.js index c58ddb789e..e785d395b0 100644 --- a/Common/3dParty/hyphen/js/test/main.js +++ b/Common/3dParty/hyphen/js/test/main.js @@ -1,40 +1,40 @@ (function (window, undefined) { - window.hyphen.loadModule(); + window.hyphen.loadModule(); - var textarea = document.getElementById("textarea"); - var form = document.querySelector("form"); - var combobox = document.getElementById("combobox"); + var textarea = document.getElementById("textarea"); + var form = document.querySelector("form"); + var combobox = document.getElementById("combobox"); - var application = undefined; - - form.onsubmit = function(event) { + var application = undefined; + + form.onsubmit = function(event) { - if(combobox.value == "") { - return; - } + if(combobox.value == "") { + return; + } - if(application == undefined) { - application = window.hyphen.hyphenCreateApplication(); - } + if(application == undefined) { + application = window.hyphen.hyphenCreateApplication(); + } - var lang = combobox.value; - var text = textarea.value.split("\n").join(" ").split(" "); - - var request = new XMLHttpRequest(); - var path = '../../../../../../dictionaries/' + lang + '/' + 'hyph_' + lang + '.dic'; - request.open('GET', path, true); - request.send(null); + var lang = combobox.value; + var text = textarea.value.split("\n").join(" ").split(" "); + + var request = new XMLHttpRequest(); + var path = '../../../../../../dictionaries/' + lang + '/' + 'hyph_' + lang + '.dic'; + request.open('GET', path, true); + request.send(null); - request.onload = function () { - var dict = request.responseText; - window.hyphen.hyphenLoadDictionary(application, dict, lang); + request.onload = function () { + var dict = request.responseText; + window.hyphen.hyphenLoadDictionary(application, dict, lang); - for(var i = 0; i < text.length; i++) { - var hyphens = window.hyphen.hyphenWord(application, text[i].toLowerCase(), lang); - console.log(hyphens); - } - } - event.preventDefault(); - } + for(var i = 0; i < text.length; i++) { + var hyphens = window.hyphen.hyphenWord(application, text[i].toLowerCase(), lang); + console.log(hyphens); + } + } + event.preventDefault(); + } })(self); \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/test/styles.css b/Common/3dParty/hyphen/js/test/styles.css index a521d17e8b..6d17ac9ed3 100644 --- a/Common/3dParty/hyphen/js/test/styles.css +++ b/Common/3dParty/hyphen/js/test/styles.css @@ -1,10 +1,10 @@ button { - width: 60px; - height: 30px; + width: 60px; + height: 30px; } #textarea{ - display: block; - width: 300px; - height: 300px; + display: block; + width: 300px; + height: 300px; } \ No newline at end of file From cb27880fd0a6da63fd141cb0af9bf2f79502505f Mon Sep 17 00:00:00 2001 From: Alexey Date: Fri, 23 Sep 2022 22:43:25 +0300 Subject: [PATCH 12/17] Update loading dictionaries --- Common/3dParty/hyphen/js/module.js | 11 ++++++----- Common/3dParty/hyphen/js/test/main.js | 10 +++++++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index b2a38470ab..cd93667658 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -79,18 +79,19 @@ return; } - var src_size = 4 * src.length + 1; + var dict = new Uint8ClampedArray(src); + var dict_size = dict.length; var lang_size = 4 * lang.length + 1; - var _src = hyphenAllocateMemory(src_size); + var _dict = hyphenAllocateMemory(dict_size); var _lang = hyphenAllocateMemory(lang_size); - Module.stringToUTF8(src, _src, src_size); + hyphenSetMemory(dict, _dict); Module.stringToUTF8(lang, _lang, lang_size); - Module._hyphenLoadDictionary(app, _src, _lang); + Module._hyphenLoadDictionary(app, _dict, _lang); - hyphenFreeMemory(_src); + hyphenFreeMemory(_dict); hyphenFreeMemory(_lang); } diff --git a/Common/3dParty/hyphen/js/test/main.js b/Common/3dParty/hyphen/js/test/main.js index e785d395b0..2ddab5d7ce 100644 --- a/Common/3dParty/hyphen/js/test/main.js +++ b/Common/3dParty/hyphen/js/test/main.js @@ -23,11 +23,19 @@ var request = new XMLHttpRequest(); var path = '../../../../../../dictionaries/' + lang + '/' + 'hyph_' + lang + '.dic'; + + request.responseType = 'arraybuffer'; + + if (request.overrideMimeType) { + request.overrideMimeType('text/plain; charset=x-user-defined'); + } else { + request.setRequestHeader('Accept-Charset', 'x-user-defined'); + } request.open('GET', path, true); request.send(null); request.onload = function () { - var dict = request.responseText; + var dict = request.response; window.hyphen.hyphenLoadDictionary(application, dict, lang); for(var i = 0; i < text.length; i++) { From 93caa3666480fc65767a972477a7fc7608255ce9 Mon Sep 17 00:00:00 2001 From: Alexey Date: Fri, 23 Sep 2022 23:29:51 +0300 Subject: [PATCH 13/17] Small refactoring --- Common/3dParty/hyphen/js/module.js | 34 +++++++++++++++++++ .../hyphen/js/src/ExportedFunctions.cpp | 6 +--- Common/3dParty/hyphen/js/src/HyphenUtils.cpp | 1 + 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index cd93667658..147a28a2ae 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -14,6 +14,39 @@ } }; + // getBinaryPromise fix? + var fetch = self.fetch; + var getBinaryPromise = null; + if (self.AscDesktopEditor && document.currentScript && 0 == document.currentScript.src.indexOf("file:///")) { + fetch = undefined; // fetch not support file:/// scheme + + getBinaryPromise = function() { + var wasmPath = "ascdesktop://fonts/" + wasmBinaryFile.substr(8); + + return new Promise(function (resolve, reject) { + var xhr = new XMLHttpRequest(); + xhr.open('GET', wasmPath, true); + xhr.responseType = 'arraybuffer'; + + if (xhr.overrideMimeType) { + xhr.overrideMimeType('text/plain; charset=x-user-defined'); + } else { + xhr.setRequestHeader('Accept-Charset', 'x-user-defined'); + } + xhr.onload = function () { + if (this.status == 200) { + resolve(new Uint8Array(this.response)); + } + }; + xhr.send(null); + }); + } + } else { + getBinaryPromise = function() { + return getBinaryPromise2(); + } + } + //desktop_fetch //polyfill @@ -134,6 +167,7 @@ // size of one symbol var symbol = size / word.length; + // if word is "broken", can returns floats for(var i = 0; i < vector.length; i++) { if(vector[i] % 2 == 1) { hyphens.push((i + 1) / symbol); diff --git a/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp b/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp index 01d345f228..a5ca6f3cc0 100644 --- a/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp +++ b/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp @@ -16,8 +16,4 @@ void hyphenLoadDictionary(CHyphenApplication *app, const char *src, const char* char* hyphenWord(CHyphenApplication *app, const char *word, const char* lang) { return app->hyphenWord(word, lang); -} - - -// emcc src/exported_functions.cpp src/HyphenUtils.cpp -o deploy/hyphen.js -sEXPORTED_FUNCTIONS='_malloc','_free','_hyphenate','_load_dictionary','_free_dictionary' -sEXPORTED_RUNTIME_METHODS='stringToUTF8','UTF8ToString' --preload-file dictionaries -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS=0 -// emcc src/exported_functions.cpp src/CHYphenApplication.cpp ../hyphen/hyphen.c ../hyphen/hnjalloc.c -o deploy/hyphen.js -sEXPORTED_FUNCTIONS='_malloc','_free','_hyphenate','_load_dictionary','_free_dictionary' -sEXPORTED_RUNTIME_METHODS='stringToUTF8','UTF8ToString' --preload-file dictionaries -sALLOW_MEMORY_GROWTH \ No newline at end of file +} \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/src/HyphenUtils.cpp b/Common/3dParty/hyphen/js/src/HyphenUtils.cpp index 3f8cfb920c..9ebbfd10b6 100644 --- a/Common/3dParty/hyphen/js/src/HyphenUtils.cpp +++ b/Common/3dParty/hyphen/js/src/HyphenUtils.cpp @@ -7,6 +7,7 @@ extern "C" #include "../../hyphen/hnjalloc.h" } +// function from hyphen.c using std::ifstream HyphenDict * hnj_hyphen_load_stream (std::istream &in) { HyphenDict *dict[2]; From c9d77f53da593bd2b750031cf4a71fa9c4a1cb87 Mon Sep 17 00:00:00 2001 From: Alexey Date: Sat, 24 Sep 2022 03:34:22 +0300 Subject: [PATCH 14/17] Hyphen update --- Common/3dParty/hyphen/js/hyphen_build/hyphen.json | 2 +- Common/3dParty/hyphen/js/library.js | 2 +- Common/3dParty/hyphen/js/module.js | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json index a5d2be5d71..b3afa9cc97 100644 --- a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json +++ b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json @@ -10,7 +10,7 @@ "compiler_flags": [ "-O3", - "-sASSERTIONS", + "-fno-exceptions", "-fno-rtti", "-Wno-unused-command-line-argument", "-sALLOW_MEMORY_GROWTH", diff --git a/Common/3dParty/hyphen/js/library.js b/Common/3dParty/hyphen/js/library.js index 2dfe3b89b1..f8bfdf047e 100644 --- a/Common/3dParty/hyphen/js/library.js +++ b/Common/3dParty/hyphen/js/library.js @@ -1,4 +1,4 @@ -(function(window, undefined) { +(function(window) { window.hyphen = window.hyphen || {}; window.hyphen.isReady = false; diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index 147a28a2ae..91a199652a 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -1,4 +1,4 @@ -(function(window, undefined) { +(function(window) { var isModuleLoaded = false; @@ -14,7 +14,8 @@ } }; - // getBinaryPromise fix? + // getBinaryPromise fix? dekstop_fetch included + /* var fetch = self.fetch; var getBinaryPromise = null; if (self.AscDesktopEditor && document.currentScript && 0 == document.currentScript.src.indexOf("file:///")) { @@ -46,6 +47,7 @@ return getBinaryPromise2(); } } + */ //desktop_fetch From 83ba081ebb61dfc09ec2d57985cc9ef422226345 Mon Sep 17 00:00:00 2001 From: Oleg Korshul Date: Sun, 25 Sep 2022 22:55:32 +0300 Subject: [PATCH 15/17] Refactoring --- Common/3dParty/hyphen/hyphen_test/main.cpp | 28 ++- Common/3dParty/hyphen/hyphen_test/test.pro | 8 +- .../3dParty/hyphen/js/hyphen_build/after.py | 5 + .../hyphen/js/hyphen_build/hyphen.json | 3 +- Common/3dParty/hyphen/js/library.js | 10 +- Common/3dParty/hyphen/js/module.js | 174 +++++------------- .../hyphen/js/src/ExportedFunctions.cpp | 6 +- .../3dParty/hyphen/js/src/ExportedFunctions.h | 4 +- .../hyphen/js/src/HyphenApplication.cpp | 19 +- .../3dParty/hyphen/js/src/HyphenApplication.h | 4 +- Common/3dParty/hyphen/js/test/main.js | 16 +- Common/js/string_utf8.js | 20 ++ 12 files changed, 136 insertions(+), 161 deletions(-) diff --git a/Common/3dParty/hyphen/hyphen_test/main.cpp b/Common/3dParty/hyphen/hyphen_test/main.cpp index b7232ce4b1..c98c37472d 100644 --- a/Common/3dParty/hyphen/hyphen_test/main.cpp +++ b/Common/3dParty/hyphen/hyphen_test/main.cpp @@ -1,7 +1,7 @@ #include #include -#include "./../hyphen/hyphen.h" +#include "./../js/src/ExportedFunctions.h" int main(int argc, char *argv[]) { @@ -10,9 +10,10 @@ int main(int argc, char *argv[]) std::string dict_filename = PRO_DIR; std::string words_filename = PRO_DIR; std::string result_filename = PRO_DIR; + std::string dict_name = "en_US"; // set your filenames here - dict_filename += "hyph_en_US.dic"; + dict_filename += ("../../../../../dictionaries/" + dict_name + "/hyph_" + dict_name + ".dic"); words_filename += "words.txt"; result_filename += "result.txt"; @@ -81,6 +82,29 @@ int main(int argc, char *argv[]) } fin.close(); fout.close(); + +#if 1 + + CHyphenApplication* pApplication = hyphenCreateApplication(); + + FILE* fDictionary = fopen(dict_filename.c_str(), "rb"); + fseek(fDictionary, 0, SEEK_END); + long lDictSize = ftell(fDictionary); + fseek(fDictionary, 0, SEEK_SET); /* same as rewind(f); */ + + char* pDictData = (char*)malloc(lDictSize); + fread(pDictData, (size_t)lDictSize, 1, fDictionary); + fclose(fDictionary); + + int nResult = hyphenLoadDictionary(pApplication, pDictData, (unsigned int)lDictSize, dict_name.c_str()); + + free(pDictData); + + char* pHyphenVector = hyphenWord(pApplication, "expedition", dict_name.c_str()); + + hyphenDestroyApplication(pApplication); + +#endif } diff --git a/Common/3dParty/hyphen/hyphen_test/test.pro b/Common/3dParty/hyphen/hyphen_test/test.pro index 787a92ef45..0cc0f126ba 100644 --- a/Common/3dParty/hyphen/hyphen_test/test.pro +++ b/Common/3dParty/hyphen/hyphen_test/test.pro @@ -19,7 +19,13 @@ DEFINES += PRO_DIR=\\\"$$PWD/\\\" HEADERS += $$PWD_ROOT_DIR/../hyphen/hyphen.h HEADERS += $$PWD_ROOT_DIR/../hyphen/hnjalloc.h -SOURCES += $$PWD_ROOT_DIR/../hyphen/hyphen.c +#SOURCES += $$PWD_ROOT_DIR/../hyphen/hyphen.c SOURCES += $$PWD_ROOT_DIR/../hyphen/hnjalloc.c +SOURCES += \ + ../js/src/ExportedFunctions.cpp \ + ../js/src/HyphenApplication.cpp + SOURCES += main.cpp + +DESTDIR = $$PWD/build diff --git a/Common/3dParty/hyphen/js/hyphen_build/after.py b/Common/3dParty/hyphen/js/hyphen_build/after.py index ba5e473953..10882877ca 100644 --- a/Common/3dParty/hyphen/js/hyphen_build/after.py +++ b/Common/3dParty/hyphen/js/hyphen_build/after.py @@ -5,5 +5,10 @@ import base base.configure_common_apps() base.replaceInFile("../deploy/engine/hyphen.js", "__ATPOSTRUN__=[];", "__ATPOSTRUN__=[onLoadModule];") base.replaceInFile("../deploy/engine/hyphen_ie.js", "__ATPOSTRUN__=[];", "__ATPOSTRUN__=[onLoadModule];") +base.replaceInFile("../deploy/engine/hyphen.js", "__ATPOSTRUN__ = [];", "__ATPOSTRUN__=[onLoadModule];") +base.replaceInFile("../deploy/engine/hyphen_ie.js", "__ATPOSTRUN__ = [];", "__ATPOSTRUN__=[onLoadModule];") + +base.replaceInFile("../deploy/engine/hyphen.js", "function getBinaryPromise()", "function getBinaryPromise2()") +base.replaceInFile("../deploy/engine/hyphen_ie.js", "function getBinaryPromise()", "function getBinaryPromise2()") base.copy_file("../library.js", "../deploy/hyphen.js") \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json index b3afa9cc97..23bd89d84b 100644 --- a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json +++ b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json @@ -13,8 +13,7 @@ "-fno-exceptions", "-fno-rtti", "-Wno-unused-command-line-argument", - "-sALLOW_MEMORY_GROWTH", - "-sEXPORTED_RUNTIME_METHODS='stringToUTF8','UTF8ToString'" + "-sALLOW_MEMORY_GROWTH" ], "exported_functions": [ "_malloc", diff --git a/Common/3dParty/hyphen/js/library.js b/Common/3dParty/hyphen/js/library.js index f8bfdf047e..59253c2252 100644 --- a/Common/3dParty/hyphen/js/library.js +++ b/Common/3dParty/hyphen/js/library.js @@ -7,17 +7,15 @@ console.log('Module is not ready'); } - window.hyphen.hyphenCreateApplication = not_ready; - window.hyphen.hyphenDestroyApplication = not_ready; - window.hyphen.hyphenLoadDictionary = not_ready; + window.hyphen.destroyApplication = not_ready; + window.hyphen.loadDictionary = not_ready; window.hyphen.hyphenWord = not_ready; window.hyphen.onLoadModule = function(exports) { window.hyphen.isReady = true; - window.hyphen.hyphenCreateApplication = exports.hyphenCreateApplication; - window.hyphen.hyphenDestroyApplication = exports.hyphenDestroyApplication; - window.hyphen.hyphenLoadDictionary = exports.hyphenLoadDictionary; + window.hyphen.destroyApplication = exports.destroyApplication; + window.hyphen.loadDictionary = exports.loadDictionary; window.hyphen.hyphenWord = exports.hyphenWord; }; diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index 91a199652a..5b9d004517 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -1,178 +1,94 @@ (function(window) { var isModuleLoaded = false; + var application; function onLoadModule() { isModuleLoaded = true; - if (window.hyphen) { + application = Module._hyphenCreateApplication(); + if (window.hyphen) { window.hyphen.onLoadModule && window.hyphen.onLoadModule({ - hyphenCreateApplication: hyphenCreateApplication, - hyphenDestroyApplication: hyphenDestroyApplication, - hyphenLoadDictionary: hyphenLoadDictionary, - hyphenWord: hyphenWord, + destroyApplication: function() { + Module._hyphenDestroyApplication(application); + }, + loadDictionary: hyphenLoadDictionary, + hyphenWord: hyphenWord }); } }; - // getBinaryPromise fix? dekstop_fetch included - /* - var fetch = self.fetch; - var getBinaryPromise = null; - if (self.AscDesktopEditor && document.currentScript && 0 == document.currentScript.src.indexOf("file:///")) { - fetch = undefined; // fetch not support file:/// scheme - - getBinaryPromise = function() { - var wasmPath = "ascdesktop://fonts/" + wasmBinaryFile.substr(8); - - return new Promise(function (resolve, reject) { - var xhr = new XMLHttpRequest(); - xhr.open('GET', wasmPath, true); - xhr.responseType = 'arraybuffer'; - - if (xhr.overrideMimeType) { - xhr.overrideMimeType('text/plain; charset=x-user-defined'); - } else { - xhr.setRequestHeader('Accept-Charset', 'x-user-defined'); - } - xhr.onload = function () { - if (this.status == 200) { - resolve(new Uint8Array(this.response)); - } - }; - xhr.send(null); - }); - } - } else { - getBinaryPromise = function() { - return getBinaryPromise2(); - } - } - */ - //desktop_fetch //polyfill + //string_utf8 + //module - /** - * @param {Number} length - * @return {Number} - * Allocate memory for wasm, returns pointer - */ - function hyphenAllocateMemory(length) { - const ptr = Module._malloc(length); - return ptr; - } - - /** - * @param {Number} ptr - * @return {void} - * Free memory - */ - function hyphenFreeMemory(ptr) { - Module._free(ptr); - } - - /** - * @param {Uint8Array} data - * @param {Number} ptr - * Fill memory - */ - function hyphenSetMemory(data, ptr) { - Module.HEAP8.set(data, ptr); - } - /** - * - * @return {Number} - * Returns pointer - */ - function hyphenCreateApplication() { - if(!isModuleLoaded) { - return; - } - return Module._hyphenCreateApplication(); - } /** * * @param {Number} app + * @param {arraybuffer} dict + * @param {String} lang + * @returns {Boolean} isSuccess */ - function hyphenDestroyApplication(app) { - if(!isModuleLoaded) { - return; - } - Module._hyphenDestroyApplication(app); - } - /** - * - * @param {Number} app - * @param {String} src - * @param {String} lang - */ - function hyphenLoadDictionary(app, src, lang) { + function hyphenLoadDictionary(dict, lang) { if(!isModuleLoaded) { return; } - var dict = new Uint8ClampedArray(src); - var dict_size = dict.length; - var lang_size = 4 * lang.length + 1; - - var _dict = hyphenAllocateMemory(dict_size); - var _lang = hyphenAllocateMemory(lang_size); + let dictSize = dict.byteLength; + let dictPointer = Module._malloc(dictSize); + Module.HEAP8.set(new Uint8ClampedArray(dict), dictPointer); - hyphenSetMemory(dict, _dict); - Module.stringToUTF8(lang, _lang, lang_size); + let langPointer = lang.toUtf8Pointer(); - Module._hyphenLoadDictionary(app, _dict, _lang); + let result = Module._hyphenLoadDictionary(application, dictPointer, dictSize, langPointer.ptr); - hyphenFreeMemory(_dict); - hyphenFreeMemory(_lang); + langPointer.free(); + Module._free(dictPointer); + + return (result === 1) ? true : false; } /** * - * @param {Number} app * @param {String} word * @param {String} lang * @returns {Uint8ClampedArray} * Returns hyphen vector of word */ - function hyphenWord(app, word, lang){ - if(!isModuleLoaded) { + function hyphenWord(word, lang) { + if (!isModuleLoaded) { return; } - var word_size = 4 * word.length + 1; - var lang_size = 4 * lang.length + 1; + let wordPointer = word.toUtf8Pointer(); + let langPointer = lang.toUtf8Pointer(); + let hyphens = []; - var _word = hyphenAllocateMemory(word_size); - var _lang = hyphenAllocateMemory(lang_size); + if (wordPointer && langPointer) { + const ptr = Module._hyphenWord(application, wordPointer.ptr, langPointer.ptr); - Module.stringToUTF8(word, _word, word_size); - Module.stringToUTF8(lang, _lang, lang_size); + wordPointer.free(); + langPointer.free(); - const ptr = Module._hyphenWord(app, _word, _lang); + var vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, 4 * word.length + 6); - hyphenFreeMemory( _word); - hyphenFreeMemory(_lang); + // calc actual size of vector + var size = 0; + for(var i = 0; i < vector.length && vector[i] != 0; i++) { + size++; + } - var hyphens = []; - var vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, 4 * word.length + 6); + // size of one symbol + var symbol = size / word.length; - // calc actual size of vector - var size = 0; - for(var i = 0; i < vector.length && vector[i] != 0; i++) { - size++; - } - - // size of one symbol - var symbol = size / word.length; - - // if word is "broken", can returns floats - for(var i = 0; i < vector.length; i++) { - if(vector[i] % 2 == 1) { - hyphens.push((i + 1) / symbol); + // if word is "broken", can returns floats + for(var i = 0; i < vector.length; i++) { + if(vector[i] % 2 == 1) { + hyphens.push((i + 1) / symbol); + } } } return hyphens; diff --git a/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp b/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp index a5ca6f3cc0..a6c155f9ba 100644 --- a/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp +++ b/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp @@ -9,11 +9,11 @@ void hyphenDestroyApplication(CHyphenApplication *app) delete app; } -void hyphenLoadDictionary(CHyphenApplication *app, const char *src, const char* lang) +int hyphenLoadDictionary(CHyphenApplication *app, const char *dict, const unsigned int dict_size, const char* lang) { - app->loadDictionary(src, lang); + return app->loadDictionary(dict, dict_size, lang); } char* hyphenWord(CHyphenApplication *app, const char *word, const char* lang) { return app->hyphenWord(word, lang); -} \ No newline at end of file +} diff --git a/Common/3dParty/hyphen/js/src/ExportedFunctions.h b/Common/3dParty/hyphen/js/src/ExportedFunctions.h index d0764b9864..cdbc3c8b43 100644 --- a/Common/3dParty/hyphen/js/src/ExportedFunctions.h +++ b/Common/3dParty/hyphen/js/src/ExportedFunctions.h @@ -11,11 +11,11 @@ extern "C" CHyphenApplication* hyphenCreateApplication(); void hyphenDestroyApplication(CHyphenApplication *app); -void hyphenLoadDictionary(CHyphenApplication *app, const char *src, const char* lang); +int hyphenLoadDictionary(CHyphenApplication *app, const char *dict, const unsigned int dict_size, const char* lang); char* hyphenWord(CHyphenApplication *app, const char *word, const char* lang); #ifdef __cplusplus } #endif // __cplusplus -#endif // EXPORTED_FUNCTIONS_H \ No newline at end of file +#endif // EXPORTED_FUNCTIONS_H diff --git a/Common/3dParty/hyphen/js/src/HyphenApplication.cpp b/Common/3dParty/hyphen/js/src/HyphenApplication.cpp index baf7c43d0b..993bb50a95 100644 --- a/Common/3dParty/hyphen/js/src/HyphenApplication.cpp +++ b/Common/3dParty/hyphen/js/src/HyphenApplication.cpp @@ -12,6 +12,11 @@ CHyphenApplication::CHyphenApplication() : } CHyphenApplication::~CHyphenApplication() { + for (std::map::iterator iter = m_mapDicts.begin(); iter != m_mapDicts.end(); iter++) + { + hnj_hyphen_free(iter->second); + } + m_mapDicts.clear(); delete[] m_pHyphenVector; } @@ -39,13 +44,13 @@ char* CHyphenApplication::hyphenWord(const char *word, const char *lang) return m_pHyphenVector; } -void CHyphenApplication::loadDictionary(const char *src, const char* lang) +int CHyphenApplication::loadDictionary(const char *dict_memory, const unsigned int dict_size, const char *lang) { std::string s_lang(lang); HyphenDict *dict; if(m_mapDicts[s_lang] != nullptr) - return; + return 0; if(m_mapDicts.size() > m_nMaxDictsCount) { @@ -54,8 +59,10 @@ void CHyphenApplication::loadDictionary(const char *src, const char* lang) m_mapDicts.erase(it); } - std::stringstream ss(src); - dict = hnj_hyphen_load_stream(ss); + std::stringstream ss; + ss.write(dict_memory, dict_size); - m_mapDicts[s_lang] = dict; -} \ No newline at end of file + m_mapDicts[s_lang] = hnj_hyphen_load_stream(ss); + + return 1; +} diff --git a/Common/3dParty/hyphen/js/src/HyphenApplication.h b/Common/3dParty/hyphen/js/src/HyphenApplication.h index f47cd63421..edaa7d15ac 100644 --- a/Common/3dParty/hyphen/js/src/HyphenApplication.h +++ b/Common/3dParty/hyphen/js/src/HyphenApplication.h @@ -15,7 +15,7 @@ public: ~CHyphenApplication(); char* hyphenWord(const char *word, const char *lang); - void loadDictionary(const char *src, const char *lang); + int loadDictionary(const char *dict, const unsigned int dict_size, const char *lang); private: std::map m_mapDicts; @@ -26,4 +26,4 @@ private: }; -#endif // HYPHEN_APPLICATION_H \ No newline at end of file +#endif // HYPHEN_APPLICATION_H diff --git a/Common/3dParty/hyphen/js/test/main.js b/Common/3dParty/hyphen/js/test/main.js index 2ddab5d7ce..40369b5f50 100644 --- a/Common/3dParty/hyphen/js/test/main.js +++ b/Common/3dParty/hyphen/js/test/main.js @@ -6,18 +6,18 @@ var form = document.querySelector("form"); var combobox = document.getElementById("combobox"); - var application = undefined; - + if (true) + { + combobox.value = "en_US"; + textarea.value = "expedition"; + } + form.onsubmit = function(event) { if(combobox.value == "") { return; } - if(application == undefined) { - application = window.hyphen.hyphenCreateApplication(); - } - var lang = combobox.value; var text = textarea.value.split("\n").join(" ").split(" "); @@ -36,10 +36,10 @@ request.onload = function () { var dict = request.response; - window.hyphen.hyphenLoadDictionary(application, dict, lang); + window.hyphen.loadDictionary(dict, lang); for(var i = 0; i < text.length; i++) { - var hyphens = window.hyphen.hyphenWord(application, text[i].toLowerCase(), lang); + var hyphens = window.hyphen.hyphenWord(text[i].toLowerCase(), lang); console.log(hyphens); } } diff --git a/Common/js/string_utf8.js b/Common/js/string_utf8.js index d03831be82..7e33fce414 100644 --- a/Common/js/string_utf8.js +++ b/Common/js/string_utf8.js @@ -117,4 +117,24 @@ return new Uint8Array(tmpStrings, 0, outputIndex); }; + function StringPointer(pointer) + { + this.ptr = pointer; + } + StringPointer.prototype.free = function() + { + if (0 !== this.ptr) + Module["_free"](this.ptr); + }; + + String.prototype.toUtf8Pointer = function(isNoEndNull) { + var tmp = this.toUtf8(isNoEndNull); + var pointer = Module["_malloc"](tmp.length); + if (0 == pointer) + return null; + + Module["HEAP8"].set(tmp, pointer); + return new StringPointer(pointer); + }; + })(); From 6f93961f67ab7ed8fe6350cd9be9cc93fc91d25f Mon Sep 17 00:00:00 2001 From: Alexey Date: Mon, 26 Sep 2022 19:38:55 +0300 Subject: [PATCH 16/17] hyphen test update --- Common/3dParty/hyphen/js/module.js | 15 ++------------- Common/3dParty/hyphen/js/test/main.js | 25 +++++++++++++++++++------ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index 5b9d004517..6add8a4a9d 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -47,7 +47,7 @@ langPointer.free(); Module._free(dictPointer); - + return (result === 1) ? true : false; } @@ -74,20 +74,9 @@ langPointer.free(); var vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, 4 * word.length + 6); - - // calc actual size of vector - var size = 0; for(var i = 0; i < vector.length && vector[i] != 0; i++) { - size++; - } - - // size of one symbol - var symbol = size / word.length; - - // if word is "broken", can returns floats - for(var i = 0; i < vector.length; i++) { if(vector[i] % 2 == 1) { - hyphens.push((i + 1) / symbol); + hyphens.push((i + 1)); } } } diff --git a/Common/3dParty/hyphen/js/test/main.js b/Common/3dParty/hyphen/js/test/main.js index 40369b5f50..922f391d7e 100644 --- a/Common/3dParty/hyphen/js/test/main.js +++ b/Common/3dParty/hyphen/js/test/main.js @@ -6,11 +6,9 @@ var form = document.querySelector("form"); var combobox = document.getElementById("combobox"); - if (true) - { - combobox.value = "en_US"; - textarea.value = "expedition"; - } + combobox.value = "en_US"; + textarea.value = "expedition"; + form.onsubmit = function(event) { @@ -40,7 +38,22 @@ for(var i = 0; i < text.length; i++) { var hyphens = window.hyphen.hyphenWord(text[i].toLowerCase(), lang); - console.log(hyphens); + console.log("In bytes: " + hyphens); + + // size of 1 symbol (1-4 bytes) + var sizeOfSym = new Blob([text[i]]).size / text[i].length; + var hword = ""; + + var lpos = 0; + for(var j = 0; j < hyphens.length; j++) { + var pos = hyphens[j] / sizeOfSym; + pos = pos | 0; + hword += text[i].substr(lpos, pos - lpos); + hword += '='; + lpos = pos; + } + hword += text[i].substr(lpos, text[i].length - lpos); + console.log(hword); } } event.preventDefault(); From fd6259fed49f42ad715e082f95933d4510b13bd3 Mon Sep 17 00:00:00 2001 From: Oleg Korshul Date: Mon, 26 Sep 2022 23:55:24 +0300 Subject: [PATCH 17/17] Refactoring --- Common/3dParty/hyphen/js/module.js | 12 ++++++------ Common/3dParty/hyphen/js/test/main.js | 25 +++++++++++++------------ Common/js/string_utf8.js | 7 ++++--- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js index 6add8a4a9d..eef6e3fe56 100644 --- a/Common/3dParty/hyphen/js/module.js +++ b/Common/3dParty/hyphen/js/module.js @@ -33,7 +33,7 @@ * @returns {Boolean} isSuccess */ function hyphenLoadDictionary(dict, lang) { - if(!isModuleLoaded) { + if (!isModuleLoaded) { return; } @@ -55,7 +55,7 @@ * * @param {String} word * @param {String} lang - * @returns {Uint8ClampedArray} + * @returns {Array} * Returns hyphen vector of word */ function hyphenWord(word, lang) { @@ -70,14 +70,14 @@ if (wordPointer && langPointer) { const ptr = Module._hyphenWord(application, wordPointer.ptr, langPointer.ptr); + let vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, wordPointer.length + 5); + wordPointer.free(); langPointer.free(); - var vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, 4 * word.length + 6); - for(var i = 0; i < vector.length && vector[i] != 0; i++) { - if(vector[i] % 2 == 1) { + for (let i = 0; vector[i] != 0; i++) { + if (1 == (vector[i] & 1)) hyphens.push((i + 1)); - } } } return hyphens; diff --git a/Common/3dParty/hyphen/js/test/main.js b/Common/3dParty/hyphen/js/test/main.js index 922f391d7e..f8914a303e 100644 --- a/Common/3dParty/hyphen/js/test/main.js +++ b/Common/3dParty/hyphen/js/test/main.js @@ -38,21 +38,22 @@ for(var i = 0; i < text.length; i++) { var hyphens = window.hyphen.hyphenWord(text[i].toLowerCase(), lang); - console.log("In bytes: " + hyphens); - // size of 1 symbol (1-4 bytes) - var sizeOfSym = new Blob([text[i]]).size / text[i].length; - var hword = ""; + let itemUtf8 = text[i].toUtf8(true); + let start = 0; + let hword = ""; - var lpos = 0; - for(var j = 0; j < hyphens.length; j++) { - var pos = hyphens[j] / sizeOfSym; - pos = pos | 0; - hword += text[i].substr(lpos, pos - lpos); - hword += '='; - lpos = pos; + for(let j = 0, len = hyphens.length; j < len; j++) { + hword += "".fromUtf8(itemUtf8, start, hyphens[j] - start); + hword += "="; + start = hyphens[j]; } - hword += text[i].substr(lpos, text[i].length - lpos); + + if (start < itemUtf8.length) { + hword += "".fromUtf8(itemUtf8, start); + hword += "="; + } + console.log(hword); } } diff --git a/Common/js/string_utf8.js b/Common/js/string_utf8.js index 7e33fce414..6483236b3f 100644 --- a/Common/js/string_utf8.js +++ b/Common/js/string_utf8.js @@ -15,7 +15,7 @@ if (undefined === start) start = 0; if (undefined === len) - len = buffer.length; + len = buffer.length - start; var result = ""; var index = start; @@ -117,9 +117,10 @@ return new Uint8Array(tmpStrings, 0, outputIndex); }; - function StringPointer(pointer) + function StringPointer(pointer, len) { this.ptr = pointer; + this.length = len; } StringPointer.prototype.free = function() { @@ -134,7 +135,7 @@ return null; Module["HEAP8"].set(tmp, pointer); - return new StringPointer(pointer); + return new StringPointer(pointer, tmp.length); }; })();