Skip to content

Commit e88abbc

Browse files
mythzmythz
committed
Add SpeechToText UI
1 parent 84162c8 commit e88abbc

File tree

2 files changed

+273
-2
lines changed

2 files changed

+273
-2
lines changed

‎AiServer/wwwroot/mjs/components/SpeechToText.mjs

Lines changed: 271 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,277 @@
1+
import { ref, computed, onMounted, inject, watch, nextTick } from "vue"
2+
import { useClient, useFiles } from "@servicestack/vue"
3+
import { createErrorStatus } from "@servicestack/client"
4+
import { SpeechToText } from "dtos"
5+
import { UiLayout, ThreadStorage, HistoryTitle, HistoryGroups, useUiLayout, icons, acceptedAudios } from "../utils.mjs"
6+
import FileUpload from "./FileUpload.mjs"
7+
18
export default {
9+
components: {
10+
UiLayout,
11+
HistoryTitle,
12+
HistoryGroups,
13+
FileUpload,
14+
},
215
template: `
3-
<h3 class="p-2 sm:block text-xl md:text-2xl font-semibold">Speech to Text</h3>
16+
<UiLayout>
17+
<template #main>
18+
<div class="flex flex-1 gap-4 text-base md:gap-5 lg:gap-6">
19+
<form ref="refForm" :disabled="client.loading.value" class="w-full mb-0" @submit.prevent="send">
20+
<div class="relative flex h-full max-w-full flex-1 flex-col">
21+
<div class="flex flex-col w-full items-center">
22+
<fieldset class="w-full">
23+
<ErrorSummary :except="visibleFields" class="mb-4" />
24+
<div class="grid grid-cols-6 gap-4">
25+
<div class="col-span-6">
26+
<FileUpload ref="refImage" id="speech" v-model="request.speech" required
27+
accept=".mp3,.m4a,.aac,.flac,.wav,.wma" :acceptLabel="acceptedAudios" @change="renderKey++">
28+
<template #title>
29+
<span class="font-semibold text-green-600">Click to upload</span> or drag and drop
30+
</template>
31+
<template #icon>
32+
<svg class="mb-2 h-12 w-12 text-green-500 inline" viewBox="0 0 24 24">
33+
<path fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1" d="M22 12c-.237 5.082-4.622 9.133-9.995 9.133q-.976.001-1.936-.178c-.459-.087-.689-.13-.849-.105c-.16.024-.387.145-.842.386a6.5 6.5 0 0 1-4.226.657a5.3 5.3 0 0 0 1.087-2.348c.1-.53-.147-1.045-.519-1.422C3.034 16.411 2 14.105 2 11.567C2 6.284 6.48 2 12.005 2q.762 0 1.495.106M16 4.5c.491-.506 1.8-2.5 2.5-2.5M21 4.5c-.491-.506-1.8-2.5-2.5-2.5m0 0v8m-6.504 2h.008m3.987 0H16m-8 0h.009" color="currentColor"/>
34+
</svg>
35+
</template>
36+
</FileUpload>
37+
</div>
38+
</div>
39+
</fieldset>
40+
</div>
41+
<div class="mt-4 mb-8 flex justify-center">
42+
<PrimaryButton :key="renderKey" type="submit" :disabled="!validPrompt()">
43+
<svg class="-ml-0.5 h-6 w-6 mr-1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path fill="currentColor" d="M11 16V7.85l-2.6 2.6L7 9l5-5l5 5l-1.4 1.45l-2.6-2.6V16zm-5 4q-.825 0-1.412-.587T4 18v-3h2v3h12v-3h2v3q0 .825-.587 1.413T18 20z"/></svg>
44+
<span class="text-base font-semibold">Upload</span>
45+
</PrimaryButton>
46+
</div>
47+
</div>
48+
</form>
49+
</div>
50+
51+
<div class="pb-20">
52+
53+
<div v-if="client.loading.value" class="mt-8 mb-20 flex justify-center items-center">
54+
<Loading class="text-gray-300 font-normal" imageClass="w-7 h-7 mt-1.5">processing audio...</Loading>
55+
</div>
56+
57+
<div v-for="result in getThreadResults()" class="w-full ">
58+
<div class="flex items-center justify-between">
59+
<span class="my-4 flex justify-center items-center text-xl underline-offset-4">
60+
<span>{{ result.request.speech }}</span>
61+
</span>
62+
<div class="group flex cursor-pointer" @click="discardResult(result)">
63+
<div class="ml-1 invisible group-hover:visible">discard</div>
64+
<svg class="w-6 h-6" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><path fill="currentColor" d="M12 12h2v12h-2zm6 0h2v12h-2z"></path><path fill="currentColor" d="M4 6v2h2v20a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V8h2V6zm4 22V8h16v20zm4-26h8v2h-8z"></path></svg>
65+
</div>
66+
</div>
67+
<div>
68+
<div v-for="output in result.response.textOutputs.filter(x => !x.text.startsWith('['))" class="relative border border-indigo-600/25 rounded-lg p-2 mb-4 overflow-hidden">
69+
<div class="prose">{{output.text}}</div>
70+
</div>
71+
</div>
72+
</div>
73+
</div>
74+
</template>
75+
76+
<template #sidebar>
77+
<HistoryTitle :prefix="storage.prefix" />
78+
<HistoryGroups :history="history" v-slot="{ item }" @save="saveHistoryItem($event)" @remove="removeHistoryItem($event)">
79+
<Icon class="h-5 w-5 rounded-full flex-shrink-0 mr-1" :src="icons.audio" loading="lazy" :alt="item.model" />
80+
<span :title="item.title">{{item.title}}</span>
81+
</HistoryGroups>
82+
</template>
83+
</UiLayout>
484
`,
585
setup() {
6-
return {}
86+
const client = useClient()
87+
const routes = inject('routes')
88+
const refUi = ref()
89+
const refForm = ref()
90+
const refImage = ref()
91+
const ui = useUiLayout(refUi)
92+
const renderKey = ref(0)
93+
const { filePathUri, getExt, extSrc, svgToDataUri } = useFiles()
94+
95+
const storage = new ThreadStorage(`spch2txt`, {
96+
tag: '',
97+
})
98+
const error = ref()
99+
100+
const prefs = ref(storage.getPrefs())
101+
const history = ref(storage.getHistory())
102+
const thread = ref()
103+
const threadRef = ref()
104+
105+
const validPrompt = () => refForm.value?.speech?.files?.length
106+
const refMessage = ref()
107+
const visibleFields = 'speech'.split(',')
108+
const request = ref(new SpeechToText())
109+
110+
function savePrefs() {
111+
storage.savePrefs(Object.assign({}, request.value, { tag:'' }))
112+
}
113+
function loadHistory() {
114+
history.value = storage.getHistory()
115+
}
116+
function saveHistory() {
117+
storage.saveHistory(history.value)
118+
}
119+
function saveThread() {
120+
if (thread.value) {
121+
storage.saveThread(thread.value)
122+
}
123+
}
124+
125+
async function send() {
126+
console.log('send', validPrompt(), client.loading.value)
127+
if (!validPrompt() || client.loading.value) return
128+
129+
savePrefs()
130+
console.debug(`${storage.prefix}.request`, Object.assign({}, request.value))
131+
132+
error.value = null
133+
let formData = new FormData(refForm.value)
134+
const speech = formData.get('speech').name
135+
136+
const api = await client.apiForm(request.value, formData, { jsconfig: 'eccn' })
137+
/** @type {GenerationResponse} */
138+
const r = api.response
139+
if (r) {
140+
console.debug(`${storage.prefix}.response`, r)
141+
142+
if (!r.textOutputs?.length) {
143+
error.value = createErrorStatus("no results were returned")
144+
} else {
145+
const id = parseInt(routes.id) || storage.createId()
146+
thread.value = thread.value ?? storage.createThread(Object.assign({
147+
id: storage.getThreadId(id),
148+
title: speech,
149+
}, request.value))
150+
151+
const result = {
152+
id: storage.createId(),
153+
request: Object.assign({}, request.value, { speech }),
154+
response: r,
155+
}
156+
thread.value.results.push(result)
157+
saveThread()
158+
159+
if (!history.value.find(x => x.id === id)) {
160+
history.value.push({
161+
id,
162+
title: thread.value.title,
163+
ext: getExt(speech),
164+
})
165+
}
166+
saveHistory()
167+
168+
if (routes.id !== id) {
169+
routes.to({ id })
170+
}
171+
}
172+
173+
} else {
174+
console.error('send.error', api.error)
175+
error.value = api.error
176+
}
177+
}
178+
179+
function getThreadResults() {
180+
const ret = Array.from(thread.value?.results ?? [])
181+
ret.reverse()
182+
return ret
183+
}
184+
185+
function selectRequest(req) {
186+
Object.assign(request.value, req)
187+
ui.scrollTop()
188+
}
189+
190+
function discardResult(result) {
191+
thread.value.results = thread.value.results.filter(x => x.id != result.id)
192+
saveThread()
193+
}
194+
195+
function toggleIcon(item) {
196+
threadRef.value.icon = item.url
197+
saveHistory()
198+
}
199+
200+
function onRouteChange() {
201+
console.log('onRouteChange', routes.id)
202+
refImage.value?.clear()
203+
loadHistory()
204+
if (routes.id) {
205+
const id = parseInt(routes.id)
206+
thread.value = storage.getThread(storage.getThreadId(id))
207+
threadRef.value = history.value.find(x => x.id === parseInt(routes.id))
208+
console.debug('thread', thread.value, threadRef.value)
209+
if (thread.value) {
210+
Object.keys(storage.defaults).forEach(k =>
211+
request.value[k] = thread.value[k] ?? storage.defaults[k])
212+
}
213+
} else {
214+
thread.value = null
215+
Object.keys(storage.defaults).forEach(k => request.value[k] = storage.defaults[k])
216+
}
217+
}
218+
219+
function updated() {
220+
onRouteChange()
221+
}
222+
223+
function saveHistoryItem(item) {
224+
storage.saveHistory(history.value)
225+
if (thread.value && item.title) {
226+
thread.value.title = item.title
227+
saveThread()
228+
}
229+
}
230+
231+
function removeHistoryItem(item) {
232+
const idx = history.value.findIndex(x => x.id === item.id)
233+
if (idx >= 0) {
234+
history.value.splice(idx, 1)
235+
storage.saveHistory(history.value)
236+
storage.deleteThread(storage.getThreadId(item.id))
237+
if (routes.id == item.id) {
238+
routes.to({ id:undefined })
239+
}
240+
}
241+
}
242+
243+
244+
watch(() => routes.id, updated)
245+
246+
onMounted(async () => {
247+
updated()
248+
})
249+
250+
return {
251+
refForm,
252+
refImage,
253+
storage,
254+
routes,
255+
client,
256+
history,
257+
request,
258+
visibleFields,
259+
validPrompt,
260+
refMessage,
261+
thread,
262+
threadRef,
263+
icons,
264+
send,
265+
saveHistory,
266+
saveThread,
267+
toggleIcon,
268+
selectRequest,
269+
discardResult,
270+
getThreadResults,
271+
saveHistoryItem,
272+
removeHistoryItem,
273+
acceptedAudios,
274+
renderKey,
275+
}
7276
}
8277
}

‎AiServer/wwwroot/mjs/utils.mjs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ export const icons = (() => {
3131
home: "viewBox='0 0 24 24' fill='none' stroke-width='1.5' stroke='currentColor' aria-hidden='true'%3E%3Cpath stroke-linecap='round' stroke-linejoin='round' d='M2.25 12l8.954-8.955c.44-.439 1.152-.439 1.591 0L21.75 12M4.5 9.75v10.125c0 .621.504 1.125 1.125 1.125H9.75v-4.875c0-.621.504-1.125 1.125-1.125h2.25c.621 0 1.125.504 1.125 1.125V21h4.125c.621 0 1.125-.504 1.125-1.125V9.75M8.25 21h8.25'%3E%3C/path%3E%3C/svg%3E",
3232
table: "viewBox='0 0 24 24' fill='none' stroke='currentColor' aria-hidden='true'%3E%3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M3 8V6a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2v2M3 8v6m0-6h6m12 0v6m0-6H9m12 6v4a2 2 0 0 1-2 2H9m12-6H9m-6 0v4a2 2 0 0 0 2 2h4m-6-6h6m0-6v6m0 0v6m6-12v12'%3E%3C/path%3E%3C/svg%3E",
3333
image: "viewBox='0 0 16 16'%3E%3Cpath fill='currentColor' d='M8 3a5 5 0 0 0-3.858 8.18l2.806-2.76a1.5 1.5 0 0 1 2.105 0l2.805 2.761A5 5 0 0 0 8 3m0 10a4.98 4.98 0 0 0 3.149-1.116L8.35 9.131a.5.5 0 0 0-.701 0l-2.798 2.754A4.98 4.98 0 0 0 8 13M2 8a6 6 0 1 1 12 0A6 6 0 0 1 2 8m8-1a1 1 0 1 0 0-2a1 1 0 0 0 0 2'/%3E%3C/svg%3E",
34+
audio: "viewBox='0 0 24 24'%3E%3Cpath fill='none' stroke='currentColor' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M2 14.959V9.04C2 8.466 2.448 8 3 8h3.586a.98.98 0 0 0 .707-.305l3-3.388c.63-.656 1.707-.191 1.707.736v13.914c0 .934-1.09 1.395-1.716.726l-2.99-3.369A.98.98 0 0 0 6.578 16H3c-.552 0-1-.466-1-1.041M16 8.5c1.333 1.778 1.333 5.222 0 7M19 5c3.988 3.808 4.012 10.217 0 14'/%3E%3C/svg%3E",
3435

3536
chat: "viewBox='0 0 26 26'%3E%3Cpath fill='currentColor' d='M10 0C4.547 0 0 3.75 0 8.5c0 2.43 1.33 4.548 3.219 6.094a4.778 4.778 0 0 1-.969 2.25a14.4 14.4 0 0 1-.656.781a2.507 2.507 0 0 0-.313.406c-.057.093-.146.197-.187.407c-.042.209.015.553.187.812l.125.219l.25.125c.875.437 1.82.36 2.688.125c.867-.236 1.701-.64 2.5-1.063c.798-.422 1.557-.864 2.156-1.187c.084-.045.138-.056.219-.094C10.796 19.543 13.684 21 16.906 21c.031.004.06 0 .094 0c1.3 0 5.5 4.294 8 2.594c.1-.399-2.198-1.4-2.313-4.375c1.957-1.383 3.22-3.44 3.22-5.719c0-3.372-2.676-6.158-6.25-7.156C18.526 2.664 14.594 0 10 0m0 2c4.547 0 8 3.05 8 6.5S14.547 15 10 15c-.812 0-1.278.332-1.938.688c-.66.355-1.417.796-2.156 1.187c-.64.338-1.25.598-1.812.781c.547-.79 1.118-1.829 1.218-3.281l.032-.563l-.469-.343C3.093 12.22 2 10.423 2 8.5C2 5.05 5.453 2 10 2'%3E%3C/path%3E%3C/svg%3E",
3637
txt2img: "viewBox='0 0 14 14'%3E%3Cg fill='none' stroke='currentColor' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M2.77 8.286A3.5 3.5 0 0 1 5.577 6.88c.818 0 1.57.28 2.166.75'/%3E%3Cpath d='M5.076 10.629h-3.5a1 1 0 0 1-1-1v-8a1 1 0 0 1 1-1h8a1 1 0 0 1 1 1v3'/%3E%3Cpath d='M5.576 5.379a1.5 1.5 0 1 0 0-3a1.5 1.5 0 0 0 0 3m1.764 5.184c-.351-.061-.351-.565 0-.626a3.18 3.18 0 0 0 2.558-2.45l.021-.097c.076-.347.57-.349.649-.003l.026.113a3.19 3.19 0 0 0 2.565 2.435c.353.062.353.568 0 .63A3.19 3.19 0 0 0 10.594 13l-.026.113c-.079.346-.573.344-.649-.003l-.021-.097a3.18 3.18 0 0 0-2.558-2.45'/%3E%3C/g%3E%3C/svg%3E",
@@ -364,6 +365,7 @@ export function wordList(items) {
364365
}
365366

366367
export const acceptedImages = `${wordList('WEBP,JPG,PNG,GIF')} (max 5MB)`
368+
export const acceptedAudios = `${wordList('MP3,M4A,AAC,FLAC,WAV,WMA')} (max 10MB)`
367369

368370
export const Img = {
369371
dataUriEscapeChars: ['%','#','<','>','?','[','\\',']','^','`','{','|','}'],

0 commit comments

Comments
 (0)