diff --git a/.gitignore b/.gitignore index 371f87f6ce..09d706f377 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ Thumbs.db .vs DesktopEditor/fontengine/js/common/freetype-2.10.4 +*_resource.rc diff --git a/Common/3dParty/hyphen/.gitignore b/Common/3dParty/hyphen/.gitignore new file mode 100644 index 0000000000..9c67b974e0 --- /dev/null +++ b/Common/3dParty/hyphen/.gitignore @@ -0,0 +1,10 @@ +hyphen +test.pro.user +*.dic +msvc_make.bat +build-test-* +*.wasm +hyphen.js +hyphen_ie.js +hyphen.data +.vscode \ No newline at end of file diff --git a/Common/3dParty/hyphen/hyphen_test/main.cpp b/Common/3dParty/hyphen/hyphen_test/main.cpp new file mode 100644 index 0000000000..c98c37472d --- /dev/null +++ b/Common/3dParty/hyphen/hyphen_test/main.cpp @@ -0,0 +1,115 @@ +#include +#include + +#include "./../js/src/ExportedFunctions.h" + +int main(int argc, char *argv[]) +{ + HyphenDict *dict; + + std::string dict_filename = PRO_DIR; + std::string words_filename = PRO_DIR; + std::string result_filename = PRO_DIR; + std::string dict_name = "en_US"; + + // set your filenames here + dict_filename += ("../../../../../dictionaries/" + dict_name + "/hyph_" + dict_name + ".dic"); + words_filename += "words.txt"; + result_filename += "result.txt"; + + // load the hyphenation dictionary + dict = hnj_hyphen_load(dict_filename.c_str()); + + std::ifstream fin(words_filename); + if(!fin.is_open()) + { + std::cerr << "could not open " << words_filename << "!" << std::endl; + return -1; + } + + std::ofstream fout(result_filename); + if(!fout.is_open()) + { + std::cerr << "could not open " << result_filename << "!" << std::endl; + return -1; + } + + while(!fin.eof()) + { + char **rep = NULL; + int *pos = NULL; + int *cut = NULL; + + std::string word; + + fin >> word; + int n = word.size(); + char *hword = new char[n * 2]; + char *hyphens = new char[n + 5]; + + /** + * @brief + * input data: + * + * word: input word + * word_size: byte length of the input word + * hyphens: allocated character buffer (size = word_size + 5) + * hyphenated_word: allocated character buffer (size ~ word_size * 2) or NULL + * rep, pos, cut: pointers (point to the allocated and _zeroed_ buffers + * (size=word_size) or with NULL value) or NULL + * + * output data: + * + * hyphens: hyphenation vector (hyphenation points signed with odd numbers). + * hyphenated_word: hyphenated input word (hyphens signed with `='). + * optional (NULL input). + * rep: NULL (only standard hyph.), or replacements (hyphenation points + * signed with `=' in replacements). + * pos: NULL, or difference of the actual position and the beginning + * positions of the change in input words. + * cut: NULL, or counts of the removed characters of the original words + * at hyphenation. + * + * Note: rep, pos, cut are complementary arrays to the hyphens, indexed with the + * character positions of the input word. + */ + hnj_hyphen_hyphenate2(dict, word.c_str(), n, hyphens, hword, &rep, &pos, &cut); + + fout << hword << ' '; + + delete[] hword; + delete[] hyphens; + } + fin.close(); + fout.close(); + +#if 1 + + CHyphenApplication* pApplication = hyphenCreateApplication(); + + FILE* fDictionary = fopen(dict_filename.c_str(), "rb"); + fseek(fDictionary, 0, SEEK_END); + long lDictSize = ftell(fDictionary); + fseek(fDictionary, 0, SEEK_SET); /* same as rewind(f); */ + + char* pDictData = (char*)malloc(lDictSize); + fread(pDictData, (size_t)lDictSize, 1, fDictionary); + fclose(fDictionary); + + int nResult = hyphenLoadDictionary(pApplication, pDictData, (unsigned int)lDictSize, dict_name.c_str()); + + free(pDictData); + + char* pHyphenVector = hyphenWord(pApplication, "expedition", dict_name.c_str()); + + hyphenDestroyApplication(pApplication); + +#endif +} + + + + + + + diff --git a/Common/3dParty/hyphen/hyphen_test/test.pro b/Common/3dParty/hyphen/hyphen_test/test.pro new file mode 100644 index 0000000000..0cc0f126ba --- /dev/null +++ b/Common/3dParty/hyphen/hyphen_test/test.pro @@ -0,0 +1,31 @@ +QT -= core +QT -= gui + +TARGET = test +CONFIG += console +CONFIG -= app_bundle + +TEMPLATE = app + +CORE_ROOT_DIR = $$PWD/../../../../../core +PWD_ROOT_DIR = $$PWD + +include($$CORE_ROOT_DIR/Common/base.pri) + +INCLUDEPATH += $$PWD_ROOT_DIR/../hyphen + +DEFINES += PRO_DIR=\\\"$$PWD/\\\" + +HEADERS += $$PWD_ROOT_DIR/../hyphen/hyphen.h +HEADERS += $$PWD_ROOT_DIR/../hyphen/hnjalloc.h + +#SOURCES += $$PWD_ROOT_DIR/../hyphen/hyphen.c +SOURCES += $$PWD_ROOT_DIR/../hyphen/hnjalloc.c + +SOURCES += \ + ../js/src/ExportedFunctions.cpp \ + ../js/src/HyphenApplication.cpp + +SOURCES += main.cpp + +DESTDIR = $$PWD/build diff --git a/Common/3dParty/hyphen/js/hyphen_build/after.py b/Common/3dParty/hyphen/js/hyphen_build/after.py new file mode 100644 index 0000000000..10882877ca --- /dev/null +++ b/Common/3dParty/hyphen/js/hyphen_build/after.py @@ -0,0 +1,14 @@ +import sys +sys.path.append("../../../../../../build_tools/scripts") +import base + +base.configure_common_apps() +base.replaceInFile("../deploy/engine/hyphen.js", "__ATPOSTRUN__=[];", "__ATPOSTRUN__=[onLoadModule];") +base.replaceInFile("../deploy/engine/hyphen_ie.js", "__ATPOSTRUN__=[];", "__ATPOSTRUN__=[onLoadModule];") +base.replaceInFile("../deploy/engine/hyphen.js", "__ATPOSTRUN__ = [];", "__ATPOSTRUN__=[onLoadModule];") +base.replaceInFile("../deploy/engine/hyphen_ie.js", "__ATPOSTRUN__ = [];", "__ATPOSTRUN__=[onLoadModule];") + +base.replaceInFile("../deploy/engine/hyphen.js", "function getBinaryPromise()", "function getBinaryPromise2()") +base.replaceInFile("../deploy/engine/hyphen_ie.js", "function getBinaryPromise()", "function getBinaryPromise2()") + +base.copy_file("../library.js", "../deploy/hyphen.js") \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/hyphen_build/hyphen.json b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json new file mode 100644 index 0000000000..23bd89d84b --- /dev/null +++ b/Common/3dParty/hyphen/js/hyphen_build/hyphen.json @@ -0,0 +1,35 @@ +{ + "name": "hyphen", + "res_folder": "../deploy/engine", + "wasm": true, + "asm": true, + "embed_mem_file": true, + "run_before": "", + "run_after": "after.py", + "base_js_content": "../module.js", + + "compiler_flags": [ + "-O3", + "-fno-exceptions", + "-fno-rtti", + "-Wno-unused-command-line-argument", + "-sALLOW_MEMORY_GROWTH" + ], + "exported_functions": [ + "_malloc", + "_free", + "_hyphenCreateApplication", + "_hyphenDestroyApplication", + "_hyphenLoadDictionary", + "_hyphenWord" + ], + "include_path": ["../src"], + "define": [], + "compile_files_array": [ + { + "name": "s", + "folder": "../src", + "files": ["ExportedFunctions.cpp", "HyphenApplication.cpp", "../../hyphen/hnjalloc.c"] + } + ] +} diff --git a/Common/3dParty/hyphen/js/library.js b/Common/3dParty/hyphen/js/library.js new file mode 100644 index 0000000000..59253c2252 --- /dev/null +++ b/Common/3dParty/hyphen/js/library.js @@ -0,0 +1,43 @@ +(function(window) { + + window.hyphen = window.hyphen || {}; + window.hyphen.isReady = false; + + var not_ready = function() { + console.log('Module is not ready'); + } + + window.hyphen.destroyApplication = not_ready; + window.hyphen.loadDictionary = not_ready; + window.hyphen.hyphenWord = not_ready; + + window.hyphen.onLoadModule = function(exports) { + window.hyphen.isReady = true; + + window.hyphen.destroyApplication = exports.destroyApplication; + window.hyphen.loadDictionary = exports.loadDictionary; + window.hyphen.hyphenWord = exports.hyphenWord; + }; + + window.hyphen.loadModule = function() { + var path = '../deploy/engine/'; + + // wasm support check + var useWasm = false; + const webAsmObj = window['WebAssembly']; + if (typeof webAsmObj === 'object') { + if (typeof webAsmObj['Memory'] === 'function') { + if ((typeof webAsmObj['instantiateStreaming'] === 'function') || (typeof webAsmObj['instantiate'] === 'function')) { + useWasm = true; + } + } + } + path += (useWasm ? 'hyphen.js' : 'hyphen_ie.js'); + + const script = document.createElement('script'); + script.type = 'text/javascript'; + script.src = path; + document.head.appendChild(script); + + } +})(self); \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/module.js b/Common/3dParty/hyphen/js/module.js new file mode 100644 index 0000000000..eef6e3fe56 --- /dev/null +++ b/Common/3dParty/hyphen/js/module.js @@ -0,0 +1,85 @@ +(function(window) { + + var isModuleLoaded = false; + var application; + + function onLoadModule() { + isModuleLoaded = true; + application = Module._hyphenCreateApplication(); + if (window.hyphen) { + window.hyphen.onLoadModule && window.hyphen.onLoadModule({ + destroyApplication: function() { + Module._hyphenDestroyApplication(application); + }, + loadDictionary: hyphenLoadDictionary, + hyphenWord: hyphenWord + }); + } + }; + + //desktop_fetch + + //polyfill + + //string_utf8 + + //module + + /** + * + * @param {Number} app + * @param {arraybuffer} dict + * @param {String} lang + * @returns {Boolean} isSuccess + */ + function hyphenLoadDictionary(dict, lang) { + if (!isModuleLoaded) { + return; + } + + let dictSize = dict.byteLength; + let dictPointer = Module._malloc(dictSize); + Module.HEAP8.set(new Uint8ClampedArray(dict), dictPointer); + + let langPointer = lang.toUtf8Pointer(); + + let result = Module._hyphenLoadDictionary(application, dictPointer, dictSize, langPointer.ptr); + + langPointer.free(); + Module._free(dictPointer); + + return (result === 1) ? true : false; + } + + /** + * + * @param {String} word + * @param {String} lang + * @returns {Array} + * Returns hyphen vector of word + */ + function hyphenWord(word, lang) { + if (!isModuleLoaded) { + return; + } + + let wordPointer = word.toUtf8Pointer(); + let langPointer = lang.toUtf8Pointer(); + let hyphens = []; + + if (wordPointer && langPointer) { + const ptr = Module._hyphenWord(application, wordPointer.ptr, langPointer.ptr); + + let vector = new Uint8ClampedArray(Module.HEAP8.buffer, ptr, wordPointer.length + 5); + + wordPointer.free(); + langPointer.free(); + + for (let i = 0; vector[i] != 0; i++) { + if (1 == (vector[i] & 1)) + hyphens.push((i + 1)); + } + } + return hyphens; + } +})(self); \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp b/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp new file mode 100644 index 0000000000..a6c155f9ba --- /dev/null +++ b/Common/3dParty/hyphen/js/src/ExportedFunctions.cpp @@ -0,0 +1,19 @@ +#include "ExportedFunctions.h" + +CHyphenApplication* hyphenCreateApplication() +{ + return new CHyphenApplication(); +} +void hyphenDestroyApplication(CHyphenApplication *app) +{ + delete app; +} + +int hyphenLoadDictionary(CHyphenApplication *app, const char *dict, const unsigned int dict_size, const char* lang) +{ + return app->loadDictionary(dict, dict_size, lang); +} +char* hyphenWord(CHyphenApplication *app, const char *word, const char* lang) +{ + return app->hyphenWord(word, lang); +} diff --git a/Common/3dParty/hyphen/js/src/ExportedFunctions.h b/Common/3dParty/hyphen/js/src/ExportedFunctions.h new file mode 100644 index 0000000000..cdbc3c8b43 --- /dev/null +++ b/Common/3dParty/hyphen/js/src/ExportedFunctions.h @@ -0,0 +1,21 @@ +#ifndef EXPORTED_FUNCTIONS_H +#define EXPORTED_FUNCTIONS_H + +#include "HyphenApplication.h" + +#ifdef __cplusplus +extern "C" +{ +#endif // __cplusplus + +CHyphenApplication* hyphenCreateApplication(); +void hyphenDestroyApplication(CHyphenApplication *app); + +int hyphenLoadDictionary(CHyphenApplication *app, const char *dict, const unsigned int dict_size, const char* lang); +char* hyphenWord(CHyphenApplication *app, const char *word, const char* lang); + +#ifdef __cplusplus +} +#endif // __cplusplus + +#endif // EXPORTED_FUNCTIONS_H diff --git a/Common/3dParty/hyphen/js/src/HyphenApplication.cpp b/Common/3dParty/hyphen/js/src/HyphenApplication.cpp new file mode 100644 index 0000000000..993bb50a95 --- /dev/null +++ b/Common/3dParty/hyphen/js/src/HyphenApplication.cpp @@ -0,0 +1,68 @@ +#include "HyphenApplication.h" + +#include "HyphenUtils.cpp" + +#define HYPHEN_VECTOR_SIZE 100 +#define MAX_DICTS_COUNT 10 + +CHyphenApplication::CHyphenApplication() : + m_nHyphenVectorSize(HYPHEN_VECTOR_SIZE), m_nMaxDictsCount(MAX_DICTS_COUNT) +{ + m_pHyphenVector = new char[m_nHyphenVectorSize]; +} +CHyphenApplication::~CHyphenApplication() +{ + for (std::map::iterator iter = m_mapDicts.begin(); iter != m_mapDicts.end(); iter++) + { + hnj_hyphen_free(iter->second); + } + m_mapDicts.clear(); + delete[] m_pHyphenVector; +} + +char* CHyphenApplication::hyphenWord(const char *word, const char *lang) +{ + std::string s_lang(lang); + HyphenDict *dict = m_mapDicts[s_lang]; + size_t n = strlen(word); + + // resize 2x + if(n + 5 > m_nHyphenVectorSize) + { + delete[] m_pHyphenVector; + m_nHyphenVectorSize *= 2; + m_pHyphenVector = new char[m_nHyphenVectorSize]; + } + + memset(m_pHyphenVector, 0, m_nHyphenVectorSize); + + char **rep = NULL; + int *pos = NULL; + int *cut = NULL; + + hnj_hyphen_hyphenate2(dict, word, n, m_pHyphenVector, NULL, &rep, &pos, &cut); + + return m_pHyphenVector; +} +int CHyphenApplication::loadDictionary(const char *dict_memory, const unsigned int dict_size, const char *lang) +{ + std::string s_lang(lang); + HyphenDict *dict; + + if(m_mapDicts[s_lang] != nullptr) + return 0; + + if(m_mapDicts.size() > m_nMaxDictsCount) + { + auto it = m_mapDicts.begin(); + hnj_hyphen_free(it->second); + m_mapDicts.erase(it); + } + + std::stringstream ss; + ss.write(dict_memory, dict_size); + + m_mapDicts[s_lang] = hnj_hyphen_load_stream(ss); + + return 1; +} diff --git a/Common/3dParty/hyphen/js/src/HyphenApplication.h b/Common/3dParty/hyphen/js/src/HyphenApplication.h new file mode 100644 index 0000000000..edaa7d15ac --- /dev/null +++ b/Common/3dParty/hyphen/js/src/HyphenApplication.h @@ -0,0 +1,29 @@ +#ifndef HYPHEN_APPLICATION_H +#define HYPHEN_APPLICATION_H + +#include +#include +#include +#include + +#include "./../../hyphen/hyphen.h" + +class CHyphenApplication +{ +public: + CHyphenApplication(); + ~CHyphenApplication(); + + char* hyphenWord(const char *word, const char *lang); + int loadDictionary(const char *dict, const unsigned int dict_size, const char *lang); + +private: + std::map m_mapDicts; + size_t m_nMaxDictsCount; + + char* m_pHyphenVector; + size_t m_nHyphenVectorSize; + +}; + +#endif // HYPHEN_APPLICATION_H diff --git a/Common/3dParty/hyphen/js/src/HyphenUtils.cpp b/Common/3dParty/hyphen/js/src/HyphenUtils.cpp new file mode 100644 index 0000000000..9ebbfd10b6 --- /dev/null +++ b/Common/3dParty/hyphen/js/src/HyphenUtils.cpp @@ -0,0 +1,189 @@ +#include +#include + +extern "C" +{ +#include "../../hyphen/hyphen.c" +#include "../../hyphen/hnjalloc.h" +} + +// function from hyphen.c using std::ifstream +HyphenDict * hnj_hyphen_load_stream (std::istream &in) +{ + HyphenDict *dict[2]; + HashTab *hashtab; + char buf[MAX_CHARS]; + int nextlevel = 0; + int i, j, k; + HashEntry *e; + int state_num = 0; + + /* loading one or two dictionaries (separated by NEXTLEVEL keyword) */ + for (k = 0; k < 2; k++) + { + hashtab = hnj_hash_new(); +#ifdef VERBOSE + global[k] = hashtab; +#endif + hnj_hash_insert (hashtab, "", 0); + dict[k] = (HyphenDict *) hnj_malloc (sizeof(HyphenDict)); + dict[k]->num_states = 1; + dict[k]->states = (HyphenState *) hnj_malloc (sizeof(HyphenState)); + dict[k]->states[0].match = NULL; + dict[k]->states[0].repl = NULL; + dict[k]->states[0].fallback_state = -1; + dict[k]->states[0].num_trans = 0; + dict[k]->states[0].trans = NULL; + dict[k]->nextlevel = NULL; + dict[k]->lhmin = 0; + dict[k]->rhmin = 0; + dict[k]->clhmin = 0; + dict[k]->crhmin = 0; + dict[k]->nohyphen = NULL; + dict[k]->nohyphenl = 0; + + /* read in character set info */ + if (k == 0) + { + for (i = 0; i < MAX_NAME; i++) + dict[k]->cset[i]= 0; + + if (in >> buf) + { + for (i = 0; i < MAX_NAME; i++) + if ((dict[k]->cset[i] == '\r') || (dict[k]->cset[i] == '\n')) + dict[k]->cset[i] = 0; + } + else + { + dict[k]->cset[0] = 0; + } + dict[k]->utf8 = (strcmp(dict[k]->cset, "UTF-8") == 0); + } + else + { + strncpy(dict[k]->cset, dict[0]->cset, sizeof(dict[k]->cset)-1); + dict[k]->cset[sizeof(dict[k]->cset)-1] = '\0'; + dict[k]->utf8 = dict[0]->utf8; + } + + if (k == 0 || nextlevel) + { + while (in.getline(buf, sizeof(buf), '\n')) + { + if(strlen(buf) < sizeof(buf)) + strcat(buf, "\n"); + + /* discard lines that don't fit in buffer */ + if (!in.eof() && strchr(buf, '\n') == NULL) + { + int c; + while ((c = in.get()) != '\n' && c != EOF); + + /* issue warning if not a comment */ + if (buf[0] != '%') + { + fprintf(stderr, "Warning: skipping too long pattern (more than %lu chars)\n", sizeof(buf)); + } + continue; + } + + if (strncmp(buf, "NEXTLEVEL", 9) == 0) + { + nextlevel = 1; + break; + } + else if (buf[0] != '%') + { + hnj_hyphen_load_line(buf, dict[k], hashtab); + } + } + } + else if (k == 1) + { + /* default first level: hyphen and ASCII apostrophe */ + if (!dict[0]->utf8) + hnj_hyphen_load_line("NOHYPHEN ',-\n", dict[k], hashtab); + + else + hnj_hyphen_load_line("NOHYPHEN ',\xe2\x80\x93,\xe2\x80\x99,-\n", dict[k], hashtab); + + strncpy(buf, "1-1\n", MAX_CHARS - 1); /* buf rewritten by hnj_hyphen_load here */ + buf[MAX_CHARS-1] = '\0'; + hnj_hyphen_load_line(buf, dict[k], hashtab); /* remove hyphen */ + hnj_hyphen_load_line("1'1\n", dict[k], hashtab); /* ASCII apostrophe */ + + if (dict[0]->utf8) + { + hnj_hyphen_load_line("1\xe2\x80\x93" "1\n", dict[k], hashtab); /* endash */ + hnj_hyphen_load_line("1\xe2\x80\x99" "1\n", dict[k], hashtab); /* apostrophe */ + } + } + + /* Could do unioning of matches here (instead of the preprocessor script). + If we did, the pseudocode would look something like this: + + foreach state in the hash table + foreach i = [1..length(state) - 1] + state to check is substr (state, i) + look it up + if found, and if there is a match, union the match in. + + It's also possible to avoid the quadratic blowup by doing the + search in order of increasing state string sizes - then you + can break the loop after finding the first match. + + This step should be optional in any case - if there is a + preprocessed rule table, it's always faster to use that. + +*/ + + /* put in the fallback states */ + for (i = 0; i < HASH_SIZE; i++) + for (e = hashtab->entries[i]; e; e = e->next) + { + if (*(e->key)) for (j = 1; 1; j++) + { + state_num = hnj_hash_lookup (hashtab, e->key + j); + if (state_num >= 0) + break; + } + /* KBH: FIXME state 0 fallback_state should always be -1? */ + if (e->val) + dict[k]->states[e->val].fallback_state = state_num; + } + +#ifdef VERBOSE + for (i = 0; i < HASH_SIZE; i++) + for (e = hashtab->entries[i]; e; e = e->next) + { + printf ("%d string %s state %d, fallback=%d\n", i, e->key, e->val, + dict[k]->states[e->val].fallback_state); + for (j = 0; j < dict[k]->states[e->val].num_trans; j++) + printf (" %c->%d\n", dict[k]->states[e->val].trans[j].ch, + dict[k]->states[e->val].trans[j].new_state); + } +#endif + +#ifndef VERBOSE + hnj_hash_free (hashtab); +#endif + state_num = 0; + } + if (nextlevel) dict[0]->nextlevel = dict[1]; + else + { + dict[1] -> nextlevel = dict[0]; + dict[1]->lhmin = dict[0]->lhmin; + dict[1]->rhmin = dict[0]->rhmin; + dict[1]->clhmin = (dict[0]->clhmin) ? dict[0]->clhmin : ((dict[0]->lhmin) ? dict[0]->lhmin : 3); + dict[1]->crhmin = (dict[0]->crhmin) ? dict[0]->crhmin : ((dict[0]->rhmin) ? dict[0]->rhmin : 3); +#ifdef VERBOSE + HashTab *r = global[0]; + global[0] = global[1]; + global[1] = r; +#endif + return dict[1]; + } + return dict[0]; +} diff --git a/Common/3dParty/hyphen/js/test/index.html b/Common/3dParty/hyphen/js/test/index.html new file mode 100644 index 0000000000..77d8bc8601 --- /dev/null +++ b/Common/3dParty/hyphen/js/test/index.html @@ -0,0 +1,52 @@ + + + + + test + + + +
+ + + +
+ + + + \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/test/main.js b/Common/3dParty/hyphen/js/test/main.js new file mode 100644 index 0000000000..f8914a303e --- /dev/null +++ b/Common/3dParty/hyphen/js/test/main.js @@ -0,0 +1,62 @@ +(function (window, undefined) { + + window.hyphen.loadModule(); + + var textarea = document.getElementById("textarea"); + var form = document.querySelector("form"); + var combobox = document.getElementById("combobox"); + + combobox.value = "en_US"; + textarea.value = "expedition"; + + + form.onsubmit = function(event) { + + if(combobox.value == "") { + return; + } + + var lang = combobox.value; + var text = textarea.value.split("\n").join(" ").split(" "); + + var request = new XMLHttpRequest(); + var path = '../../../../../../dictionaries/' + lang + '/' + 'hyph_' + lang + '.dic'; + + request.responseType = 'arraybuffer'; + + if (request.overrideMimeType) { + request.overrideMimeType('text/plain; charset=x-user-defined'); + } else { + request.setRequestHeader('Accept-Charset', 'x-user-defined'); + } + request.open('GET', path, true); + request.send(null); + + request.onload = function () { + var dict = request.response; + window.hyphen.loadDictionary(dict, lang); + + for(var i = 0; i < text.length; i++) { + var hyphens = window.hyphen.hyphenWord(text[i].toLowerCase(), lang); + + let itemUtf8 = text[i].toUtf8(true); + let start = 0; + let hword = ""; + + for(let j = 0, len = hyphens.length; j < len; j++) { + hword += "".fromUtf8(itemUtf8, start, hyphens[j] - start); + hword += "="; + start = hyphens[j]; + } + + if (start < itemUtf8.length) { + hword += "".fromUtf8(itemUtf8, start); + hword += "="; + } + + console.log(hword); + } + } + event.preventDefault(); + } +})(self); \ No newline at end of file diff --git a/Common/3dParty/hyphen/js/test/styles.css b/Common/3dParty/hyphen/js/test/styles.css new file mode 100644 index 0000000000..6d17ac9ed3 --- /dev/null +++ b/Common/3dParty/hyphen/js/test/styles.css @@ -0,0 +1,10 @@ +button { + width: 60px; + height: 30px; +} + +#textarea{ + display: block; + width: 300px; + height: 300px; +} \ No newline at end of file diff --git a/Common/js/string_utf8.js b/Common/js/string_utf8.js index d03831be82..6483236b3f 100644 --- a/Common/js/string_utf8.js +++ b/Common/js/string_utf8.js @@ -15,7 +15,7 @@ if (undefined === start) start = 0; if (undefined === len) - len = buffer.length; + len = buffer.length - start; var result = ""; var index = start; @@ -117,4 +117,25 @@ return new Uint8Array(tmpStrings, 0, outputIndex); }; + function StringPointer(pointer, len) + { + this.ptr = pointer; + this.length = len; + } + StringPointer.prototype.free = function() + { + if (0 !== this.ptr) + Module["_free"](this.ptr); + }; + + String.prototype.toUtf8Pointer = function(isNoEndNull) { + var tmp = this.toUtf8(isNoEndNull); + var pointer = Module["_malloc"](tmp.length); + if (0 == pointer) + return null; + + Module["HEAP8"].set(tmp, pointer); + return new StringPointer(pointer, tmp.length); + }; + })();