diff --git a/sdkjs-plugins/content/ai/index.html b/sdkjs-plugins/content/ai/index.html index 2fa91d83..ce3cddc8 100644 --- a/sdkjs-plugins/content/ai/index.html +++ b/sdkjs-plugins/content/ai/index.html @@ -35,6 +35,8 @@ + + diff --git a/sdkjs-plugins/content/ai/scripts/engine/engine.js b/sdkjs-plugins/content/ai/scripts/engine/engine.js index 2cf1a44e..208e2353 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/engine.js +++ b/sdkjs-plugins/content/ai/scripts/engine/engine.js @@ -96,7 +96,7 @@ }) .then(function(data) { if (data.error) - resolve({error: 1, message: data.error.message ? data.error.message : ""}); + resolve({error: 1, message: data.error.message ? data.error.message : ((typeof data.error === "string") ? data.error : "")}); else resolve({error: 0, data: data.data ? data.data : data}); }) @@ -265,10 +265,6 @@ this.errorHandler = callback; }; - AI.Request.prototype.chatRequest = async function(content, block) { - return await this._wrapRequest(this._chatRequest, content, block !== false); - }; - AI.Request.prototype._wrapRequest = async function(func, data, block) { if (block) await Asc.Editor.callMethod("StartAction", ["Block", "AI (" + this.modelUI.name + ")"]); @@ -297,6 +293,11 @@ return result; }; + // CHAT REQUESTS + AI.Request.prototype.chatRequest = async function(content, block) { + return await this._wrapRequest(this._chatRequest, content, block !== false); + }; + AI.Request.prototype._chatRequest = async function(content) { let provider = null; if (this.modelUI) @@ -476,35 +477,279 @@ } } }; - - function normalizeImageSize(size) { - let width = 0, height = 0; - if (size.width > 750 || size.height > 750) - width = height = 1024; - else if (size.width > 375 || size.height > 350) - width = height = 512; - else - width = height = 256; - return {width: width, height: height, str: width + 'x' + height} + // IMAGE REQUESTS + AI.Request.prototype.imageGenerationRequest = async function(data, block) { + return await this._wrapRequest(this._imageGenerationRequest, data, block !== false); }; - async function getImageBlob(base64) - { - return new Promise(function(resolve) { - const image = new Image(); - image.onload = function() { - const img_size = {width: image.width, height: image.height}; - const canvas_size = normalizeImageSize(img_size); - const draw_size = canvas_size.width > image.width ? img_size : canvas_size; - let canvas = document.createElement('canvas'); - canvas.width = canvas_size.width; - canvas.height = canvas_size.height; - canvas.getContext('2d').drawImage(image, 0, 0, draw_size.width, draw_size.height*image.height/image.width); - canvas.toBlob(function(blob) {resolve({blob: blob, size: canvas_size, image_size :img_size})}, 'image/png'); + AI.Request.prototype._imageGenerationRequest = async function(data) { + let provider = null; + if (this.modelUI) + provider = AI.Storage.getProvider(this.modelUI.provider); + + if (!provider) { + throw { + error : 1, + message : "Please select the correct model for action." }; - image.src = img.src; + return; + } + + let message = { + prompt : data + }; + + let objRequest = { + headers : AI._getHeaders(provider), + method : "POST", + url : AI._getEndpointUrl(provider, AI.Endpoints.Types.v1.Images_Generations, this.model), + body : provider.getImageGeneration(message, this.model) + }; + + if (objRequest.body instanceof FormData) + objRequest.isBlob = true; + + let requestBody = {}; + let processResult = function(data) { + return provider.getImageGenerationResult(data, this.model); + }; + + objRequest.isUseProxy = AI._extendBody(provider, objRequest.body); + + let result = await requestWrapper(objRequest); + if (result.error) { + throw { + error : result.error, + message : result.message + }; + return; + } + if (result.data && result.data.errors) { + throw { + error : 1, + message : result.data.errors[0] + }; + return; + } + return processResult(result); + }; + + AI.Request.prototype.imageVisionRequest = async function(data, block) { + return await this._wrapRequest(this._imageVisionRequest, data, block !== false); + }; + + AI.Request.prototype._imageVisionRequest = async function(data) { + let provider = null; + if (this.modelUI) + provider = AI.Storage.getProvider(this.modelUI.provider); + + if (!provider) { + throw { + error : 1, + message : "Please select the correct model for action." + }; + return; + } + + let message = { + prompt : data.prompt, + image : await AI.ImageEngine.getBase64FromAny(data.image) + }; + + let objRequest = { + headers : AI._getHeaders(provider), + method : "POST", + url : AI._getEndpointUrl(provider, AI.Endpoints.Types.v1.Chat_Completions, this.model), + body : provider.getImageVision(message, this.model) + }; + + if (objRequest.body instanceof FormData) + objRequest.isBlob = true; + + let requestBody = {}; + let processResult = function(data) { + return provider.getImageVisionResult(data, this.model); + }; + + objRequest.isUseProxy = AI._extendBody(provider, objRequest.body); + + let result = await requestWrapper(objRequest); + if (result.error) { + throw { + error : result.error, + message : result.message + }; + return; + } + if (result.data && result.data.errors) { + throw { + error : 1, + message : result.data.errors[0] + }; + return; + } + return processResult(result); + }; + + AI.Request.prototype.imageOCRRequest = async function(data, block) { + return await this._wrapRequest(this._imageOCRRequest, data, block !== false); + }; + + AI.Request.prototype._imageOCRRequest = async function(data) { + let provider = null; + if (this.modelUI) + provider = AI.Storage.getProvider(this.modelUI.provider); + + if (!provider) { + throw { + error : 1, + message : "Please select the correct model for action." + }; + return; + } + + let message = { + image : await AI.ImageEngine.getBase64FromAny(data) + }; + + let objRequest = { + headers : AI._getHeaders(provider), + method : "POST", + url : AI._getEndpointUrl(provider, AI.Endpoints.Types.v1.OCR, this.model), + body : provider.getImageOCR(message, this.model) + }; + + if (objRequest.body instanceof FormData) + objRequest.isBlob = true; + + let requestBody = {}; + let processResult = function(data) { + return provider.getImageOCRResult(data, this.model); + }; + + objRequest.isUseProxy = AI._extendBody(provider, objRequest.body); + + let result = await requestWrapper(objRequest); + if (result.error) { + throw { + error : result.error, + message : result.message + }; + return; + } + if (result.data && result.data.errors) { + throw { + error : 1, + message : result.data.errors[0] + }; + return; + } + return processResult(result); + }; + + AI.ImageEngine = {}; + + AI.ImageEngine.getNearestImageSize = function(w, h, sizes) { + if (!sizes) { + return { + w : sizes[i].w, + h : sizes[i].h + }; + } + + let dist = 100000; + let index = 0; + + for (let i = 0, len = sizes.length; i < len; i++) { + let tmpDist = Math.abs(w - sizes[i].w) + Math.abs(h - sizes[i].h); + if (tmpDist < dist) { + dist = tmpDist; + index = i; + } + } + + return { + w : sizes[i].w, + h : sizes[i].h + }; + }; + + AI.ImageEngine.getNearestImage = async function(input, sizes) { + let canvas = document.createElement('canvas'); + if (input instanceof HTMLImageElement || input instanceof HTMLCanvasElement) { + let dstSize = AI.ImageEngine.getNearestImageSize(input.width, input.height, sizes); + canvas.width = dstSize.w; + canvas.height = dstSize.h; + canvas.getContext('2d').drawImage(input, 0, 0, canvas.width, canvas.height); + return canvas; + } + if (image instanceof String) { + return new Promise(function(resolve) { + let image = new Image(); + image.onload = function() { + resolve(AI.ImageEngine.getNearestImage(image, sizes)); + }; + image.onerror = function() { + return resolve(null); + }; + image.src = input; + }); + } + return null; + }; + + AI.ImageEngine.getBlob = async function(canvas) { + return new Promise(function(resolve) { + var canvas_size = { + width: canvas.width, + height: canvas.height, + str: canvas.width + "x" + canvas.height + }; + canvas.toBlob(function(blob) {resolve({blob, size: canvas_size})}, 'image/png'); }); - } + }; + + AI.ImageEngine.getBase64 = function(canvas) { + return canvas.toDataURL("image/png"); + }; + + AI.ImageEngine.getBase64FromAny = async function(image) { + if (image instanceof HTMLImageElement) { + let canvas = document.createElement('canvas'); + canvas.width = image.width; + canvas.height = image.height; + canvas.getContext('2d').drawImage(image, 0, 0, canvas.width, canvas.height); + return canvas.toDataURL("image/png"); + } + if (image instanceof HTMLCanvasElement) { + return canvas.toDataURL("image/png"); + } + if (image.startsWith("data:image")) + return image; + + let canvas = await AI.ImageEngine.getNearestImage(image); + if (!canvas) + return ""; + + return AI.ImageEngine.getBase64(canvas); + }; + + AI.ImageEngine.getBase64FromUrl = async function(url) { + if (url.startsWith("data:image")) + return url; + + if (url.startsWith("iVBOR")) + return "data:image/png;base64," + url; + + if (url.startsWith("/9j/")) + return "data:image/jpeg;base64," + url; + + let canvas = await AI.ImageEngine.getNearestImage(url); + if (!canvas) + return ""; + + return AI.ImageEngine.getBase64(canvas); + }; })(window); diff --git a/sdkjs-plugins/content/ai/scripts/engine/library.js b/sdkjs-plugins/content/ai/scripts/engine/library.js index f7505899..04ec9fe1 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/library.js +++ b/sdkjs-plugins/content/ai/scripts/engine/library.js @@ -67,6 +67,22 @@ return await Editor.callMethod("GetSelectedText"); }; + Library.prototype.GetSelectedContent = async function(type) { + return await Editor.callMethod("GetSelectedContent", [{ type : type }]); + }; + + Library.prototype.GetSelectedImage = async function(type) { + let res = await Editor.callMethod("GetSelectedContent", [{ type : "html" }]); + let index1 = res.indexOf("src=\"data:image/"); + if (-1 === index1) + return ""; + index1 += 5; + let index2 = res.indexOf("\"", index1); + if (-1 === index2) + return ""; + return res.substring(index1, index2); + }; + Library.prototype.ReplaceTextSmart = async function(text) { return await Editor.callMethod("ReplaceTextSmart", [text]); @@ -88,6 +104,14 @@ }); }; + Library.prototype.InsertAsMD = async function(data) + { + let c = window.markdownit(); + let htmlContent = c.render(data); + + return await Asc.Library.InsertAsHTML(htmlContent); + }; + Library.prototype.InsertAsHTML = async function(data) { await Editor.callCommand(function() { @@ -166,6 +190,39 @@ }); }; + Library.prototype.AddGeneratedImage = async function(base64) { + switch (window.Asc.plugin.info.editorType) { + case "word": { + Asc.scope.url = base64; + return await Editor.callCommand(function() { + let document = Api.GetDocument(); + let paragraph = Api.CreateParagraph(); + let drawing = Api.CreateImage(Asc.scope.url, 25.5 * 36000, 25.5 * 36000); + paragraph.AddDrawing(drawing); + document.Push(paragraph); + }, false); + } + case "cell": { + Asc.scope.url = base64; + return await Editor.callCommand(function() { + let worksheet = Api.GetActiveSheet(); + worksheet.AddImage(Asc.scope.url, 60 * 36000, 35 * 36000, 0, 2 * 36000, 2, 3 * 36000); + }, false); + } + case "slide": { + Asc.scope.url = base64; + return await Editor.callCommand(function() { + let presentation = Api.GetPresentation(); + let slide = presentation.GetCurrentSlide(); + let image = Api.CreateImage(Asc.scope.url, 150 * 36000, 150 * 36000); + slide.AddObject(image); + }, false); + } + default: + break; + } + }; + exports.Asc = exports.Asc || {}; exports.Asc.Library = new Library(); diff --git a/sdkjs-plugins/content/ai/scripts/engine/providers/base.js b/sdkjs-plugins/content/ai/scripts/engine/providers/base.js index fc854cbe..000166f6 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/providers/base.js +++ b/sdkjs-plugins/content/ai/scripts/engine/providers/base.js @@ -82,7 +82,9 @@ Realtime : 0x51, Language : 0x61, - Code : 0x62 + Code : 0x62, + + OCR : 0x70 } } diff --git a/sdkjs-plugins/content/ai/scripts/engine/providers/config.json b/sdkjs-plugins/content/ai/scripts/engine/providers/config.json index a5001834..7a8b519e 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/providers/config.json +++ b/sdkjs-plugins/content/ai/scripts/engine/providers/config.json @@ -8,5 +8,6 @@ "ollama", "mistral", "gpt4all", - "xAI" + "xAI", + "stabilityai" ] diff --git a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/anthropic.js b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/anthropic.js index 3f163f0e..1d8e6e42 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/anthropic.js +++ b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/anthropic.js @@ -27,8 +27,18 @@ class Provider extends AI.Provider { } getEndpointUrl(endpoint, model) { - if (AI.Endpoints.Types.v1.Chat_Completions === endpoint) - return "/messages"; + switch (endpoint) + { + case AI.Endpoints.Types.v1.Chat_Completions: + case AI.Endpoints.Types.v1.Images_Generations: + case AI.Endpoints.Types.v1.Images_Edits: + case AI.Endpoints.Types.v1.Images_Variarions: + { + return "/messages"; + } + default: + break; + } return super.getEndpointUrl(endpoint, model); } @@ -58,4 +68,8 @@ class Provider extends AI.Provider { return result; } + getImageGeneration(message, model) { + return this.getImageGenerationWithChat(message, model, "Image must be in svg format. "); + } + } diff --git a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/google-gemini.js b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/google-gemini.js index eaf80711..dd03f7d9 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/google-gemini.js +++ b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/google-gemini.js @@ -58,7 +58,12 @@ class Provider extends AI.Provider { url = "/models"; break; default: - url = "/" + model.id + ":generateContent"; + let addon = ":generateContent"; + if (endpoint === Types.v1.Images_Generations) { + if (-1 != model.id.indexOf("imagen-3")) + addon = ":predict"; + } + url = "/" + model.id + addon; break; } if (this.key) @@ -91,4 +96,26 @@ class Provider extends AI.Provider { return body; } + getImageGeneration(message, model) { + if (-1 != model.id.indexOf("flash")) { + let result = this.getImageGenerationWithChat(message, model); + result.generationConfig = {"responseModalities":["TEXT","IMAGE"]}; + return result; + } + if (-1 != model.id.indexOf("imagen-3")) { + return { + instances: [ + { + prompt: message.prompt + } + ], + parameters: { + "sampleCount": 1 + } + }; + } + + return {}; + } + } diff --git a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/mistral.js b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/mistral.js index 038e2036..64657372 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/mistral.js +++ b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/mistral.js @@ -55,4 +55,61 @@ class Provider extends AI.Provider { return capUI; } + getEndpointUrl(endpoint, model) { + let Types = AI.Endpoints.Types; + let url = ""; + switch (endpoint) + { + case Types.v1.OCR: + url = "/ocr"; + break; + default: + break; + } + if (!url) + return super.getEndpointUrl(endpoint, model); + return url; + } + + getImageOCR(message, model) { + let result = { + model: model.id, + document: { + type: "image_url", + image_url: message.image + } + }; + //result.output_format = "markdown"; + result.include_image_base64 = true; + return result; + } + + getImageOCRResult(messageInput, model) { + let message = messageInput.data ? messageInput.data : messageInput; + let images = []; + let markdownContent = ""; + if (!message.pages) + return markdownContent; + + for (let i = 0, len = message.pages.length; i < len; i++) { + let page = message.pages[i]; + + let images = page.images; + let md = page.markdown; + + for (let j = 0, imagesCount = images.length; j < imagesCount; j++) { + let src = "](" + images[j].id + ")"; + let dst = "](" + images[j].image_base64 + ")"; + + src = src.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + md = md.replace(new RegExp(src, "g"), dst); + } + + markdownContent += md; + markdownContent += "\n\n"; + } + + return markdownContent; + } + } diff --git a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/ollama.js b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/ollama.js index 6951c28b..93e44a4f 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/ollama.js +++ b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/ollama.js @@ -6,4 +6,17 @@ class Provider extends AI.Provider { super("Ollama", "http://localhost:11434", "", "v1"); } + getImageGeneration(message, model) { + let result = super.getImageGeneration(message, model); + result.options = {}; + if (result.width) + result.options.width = result.width; + if (result.height) + result.options.height = result.height; + delete result.width; + delete result.height; + delete result.n; + return result; + } + } diff --git a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/openai.js b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/openai.js index 4a32e20a..46e2b9b1 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/openai.js +++ b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/openai.js @@ -78,4 +78,12 @@ class Provider extends AI.Provider { return AI.CapabilitiesUI.Chat | AI.CapabilitiesUI.Vision; }; + getImageGeneration(message, model) { + let result = super.getImageGeneration(message, model); + result.size = result.width + "x" + result.height; + delete result.width; + delete result.height; + return result; + } + } diff --git a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/stabilityai.js b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/stabilityai.js new file mode 100644 index 00000000..9ba7fba4 --- /dev/null +++ b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/stabilityai.js @@ -0,0 +1,64 @@ +"use strict"; + +class Provider extends AI.Provider { + + constructor() { + super("Stability AI", "https://api.stability.ai", "", ""); + } + + getModels() { + return [ + { + id: "Stable Diffusion" + }, + { + id: "Stable Image Core" + }, + { + id: "Stable Image Ultra" + } + ]; + } + + checkModelCapability(model) { + model.endpoints.push(AI.Endpoints.Types.v1.Images_Generations); + return AI.CapabilitiesUI.Image; + }; + + getImageGeneration(message, model) { + let formData = new FormData(); + formData.append("prompt", message.prompt); + formData.append("output_format", "png"); + return formData; + } + + getEndpointUrl(endpoint, model) { + let Types = AI.Endpoints.Types; + let url = ""; + switch (endpoint) + { + case Types.v1.Images_Generations: + if (model.id === "Stable Diffusion") + return "/v2beta/stable-image/generate/sd3"; + if (model.id === "Stable Image Core") + return "/v2beta/stable-image/generate/core"; + if (model.id === "Stable Image Ultra") + return "/v2beta/stable-image/generate/ultra"; + break; + default: + break; + } + + return super.getEndpointUrl(endpoint, model); + } + + getRequestHeaderOptions() { + let headers = { + "Accept": "application/json" + }; + if (this.key) + headers["Authorization"] = "Bearer " + this.key; + return headers; + } + +} diff --git a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/xAI.js b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/xAI.js index c5c90d98..5c877d10 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/providers/internal/xAI.js +++ b/sdkjs-plugins/content/ai/scripts/engine/providers/internal/xAI.js @@ -20,6 +20,12 @@ class Provider extends AI.Provider { return AI.CapabilitiesUI.Chat | AI.CapabilitiesUI.Vision; } + if (-1 != model.id.indexOf("image")) + { + model.endpoints.push(AI.Endpoints.Types.v1.Image_Generation | AI.Endpoints.Types.v1.Images_Edits); + return AI.CapabilitiesUI.Image; + } + model.options.max_input_tokens = AI.InputMaxTokens["128k"]; model.endpoints.push(AI.Endpoints.Types.v1.Chat_Completions); return AI.CapabilitiesUI.Chat; diff --git a/sdkjs-plugins/content/ai/scripts/engine/providers/provider.js b/sdkjs-plugins/content/ai/scripts/engine/providers/provider.js index a8122b20..1b9a8ff3 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/providers/provider.js +++ b/sdkjs-plugins/content/ai/scripts/engine/providers/provider.js @@ -101,6 +101,9 @@ case Types.v1.Realtime: return "/realtime"; + case Types.v1.OCR: + return "/chat/completions"; + default: break; } @@ -230,12 +233,162 @@ iEnd--; if (iEnd > iStart && ((0 !== iStart) || ((result.content[i].length - 1) !== iEnd))) - result.content[i] = result.content[i].substring(iStart, iEnd + 1); + result.content[i] = result.content[i].substring(iStart, iEnd + 1); } return result; } + /** + * Get available sizes for input images. + * @returns {Array.} sizes + */ + getImageSizesInput(model) { + return [ + { w: 256, h: 256 }, + { w: 512, h: 512 }, + { w: 1024, h: 1024 } + ]; + } + + /** + * Get available sizes for outpit images. + * @returns {Array.} sizes + */ + getImageSizesOutput(model) { + return [ + { w: 256, h: 256 }, + { w: 512, h: 512 }, + { w: 1024, h: 1024 } + ]; + } + + /** + * Get request body object by message. + * @param {Object} message + * *message* is in folowing format: + * { + * prompt: "", + * width:1024, + * height:1024, + * background: "transparent", + * quality: "high" + * } + */ + getImageGeneration(message, model) { + let sizes = this.getImageSizesOutput(model); + let index = sizes.length - 1; + + return { + model : model.id, + width : message.width || sizes[index].w, + height : message.width || sizes[index].h, + n : 1, + response_format : "b64_json", + prompt : message.prompt + }; + } + + /** + * Convert *getImageGeneration* answer to result base64 image. + * @returns {String} Image in base64 format + */ + async getImageGenerationResult(message, model) { + let imageUrl = ""; + let getProp = function(name) { + if (message[name]) + return message[name]; + if (message.data && message.data[name]) + return message.data[name]; + return undefined; + }; + + if (!imageUrl) { + let data = getProp("data"); + if (data && data[0] && data[0].b64_json) + imageUrl = data[0].b64_json; + } + + if (!imageUrl) { + let artifacts = getProp("artifacts"); + if (artifacts && artifacts[0] && artifacts[0].base64) + imageUrl = artifacts[0].base64; + } + + if (!imageUrl) { + let result = getProp("result"); + if (result && result.imageUrl) + imageUrl = result.imageUrl; + } + + if (!imageUrl) { + let generations = getProp("generations"); + if (generations && generations[0] && generations[0].url) + imageUrl = generations[0].url; + } + + if (!imageUrl) { + let candidates = getProp("candidates"); + if (candidates && candidates[0] && candidates[0].content) + imageUrl = candidates[0].content; + } + + if (!imageUrl) { + let image = getProp("image"); + if (image) + imageUrl = image; + } + + if (!imageUrl) { + let response = getProp("response"); + if (response) { + let matches = response.match(/data:image\/[^;]+;base64,([^"'\s]+)/); + if (matches && matches[1]) + imageUrl = matches[1]; + } + } + + if (!imageUrl) { + let content = getProp("content"); + if (content) { + for (let i = 0, len = content.length; i < len; i++) { + if (content[i].type === 'text') { + let svgMatch = content[i].text.match(//); + if (svgMatch) { + imageUrl = svgMatch[0]; + break; + } + } + } + } + + if (imageUrl) { + imageUrl = "data:image/svg+xml;base64," + btoa(imageUrl); + } + } + + if (!imageUrl) + return ""; + + return await AI.ImageEngine.getBase64FromUrl(imageUrl); + } + + async getImageVision() { + return {}; + } + + async getImageVisionResult() { + return {}; + } + + async getImageOCR() { + return {}; + } + + async getImageOCRResult() { + return {}; + } + /** * ======================================================================================== * The following are methods for internal work. There is no need to overload these methods. @@ -281,6 +434,28 @@ } return result; } + + getImageGenerationWithChat(message, model, addon) { + let prompt = "Please generate image. "; + if (addon) + prompt += addon; + // TODO: sizes + prompt += "Here is the description for the image content:\""; + prompt += message.prompt; + prompt += "\""; + + let data = { + messages : [ + { + role: "user", + content: prompt + } + ] + }; + + return this.getChatCompletions(data, model); + } + } window.AI.Provider = Provider; diff --git a/sdkjs-plugins/content/ai/scripts/engine/register.js b/sdkjs-plugins/content/ai/scripts/engine/register.js index 8d673550..59a478bf 100644 --- a/sdkjs-plugins/content/ai/scripts/engine/register.js +++ b/sdkjs-plugins/content/ai/scripts/engine/register.js @@ -449,6 +449,50 @@ }); } + if (true) + { + let buttonImages = new Asc.ButtonContextMenu(buttonMain); + buttonImages.text = "Image"; + buttonImages.addCheckers("Selection", "Image"); + buttonImages.separator = true; + + let buttonGen = new Asc.ButtonContextMenu(buttonImages); + buttonGen.text = "Generate by selection text"; + buttonGen.addCheckers("Selection"); + buttonGen.attachOnClick(async function(){ + let requestEngine = AI.Request.create(AI.ActionType.ImageGeneration); + if (!requestEngine) + return; + + let content = await Asc.Library.GetSelectedText(); + if (!content) + return; + + let result = await requestEngine.imageGenerationRequest(content); + if (!result) return; + + await Asc.Library.AddGeneratedImage(result); + }); + + let buttonOCR = new Asc.ButtonContextMenu(buttonImages); + buttonOCR.text = "OCR"; + buttonOCR.addCheckers("Image"); + buttonOCR.attachOnClick(async function(){ + let requestEngine = AI.Request.create(AI.ActionType.OCR); + if (!requestEngine) + return; + + let content = await Asc.Library.GetSelectedImage(); + if (!content) + return; + + let result = await requestEngine.imageOCRRequest(content); + if (!result) return; + + await Asc.Library.InsertAsMD(result); + }); + } + if (true) { let button1 = new Asc.ButtonContextMenu(buttonMain); @@ -562,7 +606,9 @@ Summarization : "Summarization", //Text2Image : "Text2Image", Translation : "Translation", - TextAnalyze : "TextAnalyze" + TextAnalyze : "TextAnalyze", + ImageGeneration : "ImageGeneration", + OCR : "OCR" }; AI.Actions = {}; @@ -574,11 +620,13 @@ this.capabilities = (capabilities === undefined) ? AI.CapabilitiesUI.Chat : capabilities; } - AI.Actions[AI.ActionType.Chat] = new ActionUI("Ask AI", "ask-ai"); - AI.Actions[AI.ActionType.Summarization] = new ActionUI("Summarization", "summarization"); - //AI.Actions[AI.ActionType.Text2Image] = new ActionUI("Text to image", "text-to-image", "", AI.CapabilitiesUI.Image); - AI.Actions[AI.ActionType.Translation] = new ActionUI("Translation", "translation"); - AI.Actions[AI.ActionType.TextAnalyze] = new ActionUI("Text analysis", ""); + AI.Actions[AI.ActionType.Chat] = new ActionUI("Ask AI", "ask-ai"); + AI.Actions[AI.ActionType.Summarization] = new ActionUI("Summarization", "summarization"); + //AI.Actions[AI.ActionType.Text2Image] = new ActionUI("Text to image", "text-to-image", "", AI.CapabilitiesUI.Image); + AI.Actions[AI.ActionType.Translation] = new ActionUI("Translation", "translation"); + AI.Actions[AI.ActionType.TextAnalyze] = new ActionUI("Text analysis", ""); + AI.Actions[AI.ActionType.ImageGeneration] = new ActionUI("Image generation", "", "", AI.CapabilitiesUI.Image); + AI.Actions[AI.ActionType.OCR] = new ActionUI("OCR", "", "", AI.CapabilitiesUI.Vision); AI.ActionsGetKeys = function() { @@ -587,7 +635,9 @@ AI.ActionType.Summarization, //AI.ActionType.Text2Image, AI.ActionType.Translation, - AI.ActionType.TextAnalyze + AI.ActionType.TextAnalyze, + AI.ActionType.ImageGeneration, + AI.ActionType.OCR ]; };