r0r-5chach
/
mia
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ui editing

This commit is contained in:
Joshua Perry 2024-10-26 19:07:40 +01:00
parent 6d5ba0beea
commit 66f696567c
16 changed files with 409 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.gradle.kts
View File

@ -58,6 +58,10 @@ dependencies {
// Logging
implementation("io.github.oshai:kotlin-logging-jvm:7.0.0")
implementation("ch.qos.logback:logback-classic:1.5.11")
//Json
implementation("org.jetbrains.kotlinx:kotlinx-serialization-core:1.5.1")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.5.1")
}
compose.desktop {

18
src/main/kotlin/Main.kt
View File

@ -1,12 +1,20 @@
import androidx.compose.foundation.isSystemInDarkTheme
import androidx.compose.foundation.layout.WindowInsets
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.material.MaterialTheme
import androidx.compose.material.Scaffold
import androidx.compose.material.darkColors
import androidx.compose.material.lightColors
import androidx.compose.ui.Modifier
import androidx.compose.ui.unit.dp
import androidx.compose.ui.window.Window
import androidx.compose.ui.window.application
import cafe.adriel.voyager.navigator.CurrentScreen
import cafe.adriel.voyager.navigator.Navigator
import io.github.oshai.kotlinlogging.KotlinLogging
import ui.screens.Home
import ui.TopBar
import ui.Bars
import java.io.File
import java.nio.file.Paths
import kotlin.io.path.absolutePathString
@ -17,12 +25,14 @@ fun main() = application {
logger.info { "\n================== NEW APPLICATION SESSION ==================" }
checkLogsDir()
Window(onCloseRequest = ::exitApplication) {
MaterialTheme {
MaterialTheme(
colors = if (isSystemInDarkTheme()) darkColors() else lightColors()
) {
Navigator(Home()) { navigator ->
Scaffold(
topBar = { TopBar.topBar(navigator.size) },
topBar = { Bars.topBar(navigator.size) },
content = { CurrentScreen() },
bottomBar = { }
bottomBar = { Bars.bottomBar() },
)
}

10
src/main/kotlin/Microphone.kt → src/main/kotlin/processing/Microphone.kt
View File

@ -1,11 +1,13 @@
package processing
import kotlinx.coroutines.DelicateCoroutinesApi
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.async
import kotlinx.serialization.json.Json
import javax.sound.sampled.AudioFormat
import javax.sound.sampled.AudioSystem
import javax.sound.sampled.DataLine
import javax.sound.sampled.TargetDataLine
import processing.STT
import ui.Capture.Companion.captureLogger
/**
@ -26,7 +28,7 @@ class Microphone {
*/
@OptIn(DelicateCoroutinesApi::class)
fun startCapture(captureDuration: Int) {
captureLogger.info { "\n================== NEW CAPTURE SESSION ==================" }
captureLogger.info { "{NEW CAPTURE SESSION}" }
capturing = true
source.open(audioFormat, source.bufferSize)
source.start()
@ -38,7 +40,9 @@ class Microphone {
while (capturing) {
val audioBytes = source.read(buffer, 0, buffer.size)
val result = stt.parseBuffer(buffer, audioBytes)
captureLogger.info { ("Captured Speech: $result") }
if (result.isNotEmpty()) {
captureLogger.info { result }
}
//TODO: Pass onto processing
}
stt.closeModel()

5
src/main/kotlin/processing/STT.kt
View File

@ -1,5 +1,6 @@
package processing
import kotlinx.serialization.json.Json
import logger
import org.vosk.Model
import org.vosk.Recognizer
@ -10,10 +11,11 @@ import kotlin.io.path.absolutePathString
* TODO: Documentation
*/
class STT {
val recognizer = getRecognizer()
/**
*
*/
fun parseBuffer(buffer: ByteArray, audioBytes: Int): String? = if (recognizer.acceptWaveForm(buffer, audioBytes)) recognizer.result else recognizer.partialResult
fun parseBuffer(buffer: ByteArray, audioBytes: Int): String = if (recognizer.acceptWaveForm(buffer, audioBytes)) recognizer.result.split("\"")[3] else recognizer.partialResult.split("\"")[3]
fun closeModel() {
recognizer.close()
@ -27,7 +29,6 @@ class STT {
/**
*
*/
private val recognizer = getRecognizer()
/**
*

8
src/main/kotlin/processing/SemanticSimilarity.kt
View File

@ -15,15 +15,15 @@ import java.nio.file.Paths
class SemanticSimilarity {
companion object {
val manager = NDManager.newBaseManager()
val model = getModel()
val tokenizer = getTokenizer()
val model = initModel()
val tokenizer = initTokenizer()
fun getModel(): Predictor<NDList, NDList> {
fun initModel(): Predictor<NDList, NDList> {
logger.info { "Semantic similarity model initialized" }
return getCriteria().loadModel().newPredictor()
}
fun getTokenizer(): HuggingFaceTokenizer {
fun initTokenizer(): HuggingFaceTokenizer {
logger.info { "Semantic similarity tokenizer initialized" }
return HuggingFaceTokenizer.newInstance("sentence-transformers/all-mpnet-base-v2")
}

96
src/main/kotlin/ui/Bars.kt Normal file
View File

@ -0,0 +1,96 @@
package ui
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.WindowInsets
import androidx.compose.foundation.layout.fillMaxHeight
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.foundation.layout.wrapContentHeight
import androidx.compose.foundation.layout.wrapContentWidth
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.Button
import androidx.compose.material.ButtonDefaults
import androidx.compose.material.Icon
import androidx.compose.material.MaterialTheme
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ArrowBackIosNew
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.shadow
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.unit.dp
import androidx.constraintlayout.compose.ConstraintLayout
import androidx.constraintlayout.compose.Dimension
import cafe.adriel.voyager.navigator.LocalNavigator
import cafe.adriel.voyager.navigator.currentOrThrow
import ui.screens.Settings
class Bars {
companion object {
@Composable
fun topBar(screenCount: Int) {
ConstraintLayout(
Modifier
.fillMaxWidth()
.padding(10.dp)
) {
val (back, capture) = createRefs()
if (screenCount > 1) {
backButton(Modifier.constrainAs(back) {
linkTo(parent.top, parent.bottom)
start.linkTo(parent.start, margin = 5.dp)
})
}
Capture.captureContext(Modifier.constrainAs(capture) {
linkTo(parent.top, parent.bottom)
linkTo(parent.start, parent.end)
centerVerticallyTo(parent)
})
}
}
@Composable
fun bottomBar() {
ConstraintLayout(
Modifier
.padding(5.dp)
.fillMaxWidth()
) {
val currentProfile = createRef()
Profiles.currentProfile(Modifier.constrainAs(currentProfile) {
bottom.linkTo(parent.bottom)
start.linkTo(parent.start)
})
}
}
@Composable
fun backButton(modifier: Modifier) {
val navigator = LocalNavigator.currentOrThrow
Box(
modifier = modifier
.background(MaterialTheme.colors.primary, CircleShape)
) {
Button(
shape = CircleShape,
colors = ButtonDefaults.buttonColors(backgroundColor = Color.Transparent, contentColor = MaterialTheme.colors.onPrimary),
elevation = ButtonDefaults.elevation(0.dp),
onClick = {
navigator.pop()
}) {
Icon(
imageVector = Icons.Filled.ArrowBackIosNew,
contentDescription = "Go Back"
)
}
}
}
}
}

130
src/main/kotlin/ui/Capture.kt
View File

@ -1,26 +1,52 @@
package ui
import Microphone
import androidx.compose.foundation.background
import androidx.compose.foundation.horizontalScroll
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxHeight
import processing.Microphone
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.foundation.verticalScroll
import androidx.compose.material.Button
import androidx.compose.material.ButtonDefaults
import androidx.compose.material.Icon
import androidx.compose.material.MaterialTheme
import androidx.compose.material.Slider
import androidx.compose.material.Text
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.PlayArrow
import androidx.compose.material.icons.filled.Stop
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.draw.shadow
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.constraintlayout.compose.ConstraintLayout
import io.github.oshai.kotlinlogging.KotlinLogging
import kotlinx.coroutines.DelicateCoroutinesApi
import kotlinx.coroutines.delay
import logger
import ui.screens.Settings
import ui.screens.Settings.Companion.userSettings
import java.io.File
import java.nio.file.Paths
import kotlin.io.path.absolutePathString
import kotlin.math.max
import kotlin.math.roundToInt
class Capture {
@ -32,21 +58,41 @@ class Capture {
fun getCaptureOnClick(mic: Microphone) = if (!mic.capturing) mic.startCapture(getCaptureDuration()) else mic.stopCapture()
fun getCaptureIcon(capturing: Boolean) = if (!capturing) Icons.Filled.PlayArrow else Icons.Filled.Stop
fun getCaptureDescription(capturing: Boolean) = if (!capturing) "Start" else "Stop"
fun getCaptureLog() = File(Paths.get("logs/capture.log").absolutePathString()).readLines()
@Composable
fun captureContext(modifier: Modifier) {
Column(modifier) {
captureButton()
lastCapture()
ConstraintLayout(
modifier = modifier
.background(MaterialTheme.colors.primary, CircleShape)
) {
val (button, info, settings) = createRefs()
captureButton(Modifier.constrainAs(button) {
linkTo(parent.top, parent.bottom)
linkTo(parent.start, info.start, startMargin = 5.dp)
})
lastCapture(Modifier.constrainAs(info) {
linkTo(parent.top, parent.bottom)
linkTo(button.end, settings.start, startMargin = 5.dp)
})
Settings.settingsButton(Modifier.constrainAs(settings) {
linkTo(parent.top, parent.bottom)
linkTo(info.end, parent.end, startMargin = 5.dp, endMargin = 5.dp)
})
}
}
@OptIn(DelicateCoroutinesApi::class)
@Composable
fun captureButton() {
fun captureButton(modifier: Modifier) {
val mic = Microphone()
var isCapturing by rememberSaveable { mutableStateOf(mic.capturing) }
Button(
shape = CircleShape,
colors = ButtonDefaults.buttonColors(contentColor = MaterialTheme.colors.onPrimary, backgroundColor = Color.Transparent),
elevation = ButtonDefaults.elevation(0.dp),
modifier = modifier,
onClick = {
getCaptureOnClick(mic)
isCapturing = !isCapturing
@ -67,20 +113,90 @@ class Capture {
valueRange = 1.0f..20.0f,
value = sliderPosition,
onValueChange = {
if (sliderPosition != it) {
sliderPosition = it.roundToInt().toFloat()
userSettings.setProperty("capture_duration", "${it.roundToInt()}")
userSettings.save()
logger.info { "User Setting Changed: capture_duration = ${it.roundToInt()}" }
}
}
)
Text("$sliderPosition seconds")
}
@Composable
fun captureLog() {} //TODO: Display capture.{date}.log to screen
fun captureLog() {
var lines by rememberSaveable { mutableStateOf(getCaptureLog())}
Column(
modifier = Modifier
.fillMaxWidth()
.padding(start = 25.dp, end = 25.dp, top = 10.dp, bottom = 50.dp)
.verticalScroll(rememberScrollState())
) {
var count = 1
Text(
color = MaterialTheme.colors.onBackground,
text = "Capture Log:",
modifier = Modifier
.background(MaterialTheme.colors.onBackground.copy(alpha = 0.05f), CircleShape)
.padding(start = 5.dp)
.fillMaxWidth()
)
lines.forEach { line ->
val backgroundColor = if (count % 2 == 0) MaterialTheme.colors.onBackground.copy(alpha = 0.05f) else MaterialTheme.colors.background
Text(
color = MaterialTheme.colors.onBackground,
text = line,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
modifier = Modifier
.background(backgroundColor, CircleShape)
.padding(start = 5.dp)
.fillMaxWidth()
)
count += 1
}
}
LaunchedEffect(Unit) {
while(true) {
val newLines = getCaptureLog()
if (newLines != lines) {
lines = newLines
}
delay(getCaptureDuration().toLong() * 1100)
}
}
}
@Composable
fun lastCapture() {} //TODO: Display capture.{date}.log last added line
fun lastCapture(modifier: Modifier) { //TODO: Constrain size so buttons don't move everywhere
var lines by rememberSaveable { mutableStateOf(getCaptureLog()) }
Text(
color = MaterialTheme.colors.onPrimary,
modifier = modifier
.width(300.dp),
text = if (lines.isNotEmpty()) lines.last() else "No captures this session",
overflow = TextOverflow.Ellipsis,
textAlign = TextAlign.Center,
maxLines = 1
)
LaunchedEffect(Unit) {
while(true) {
val newLines = getCaptureLog()
if (newLines != lines) {
lines = newLines
}
delay((getCaptureDuration().toLong() + 1) * 1000)
}
}
}
}
}

101
src/main/kotlin/ui/Profiles.kt Normal file
View File

@ -0,0 +1,101 @@
package ui
import androidx.compose.foundation.ExperimentalFoundationApi
import androidx.compose.foundation.background
import androidx.compose.foundation.clickable
import androidx.compose.foundation.combinedClickable
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.material.MaterialTheme
import androidx.compose.material.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import cafe.adriel.voyager.navigator.LocalNavigator
import cafe.adriel.voyager.navigator.currentOrThrow
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.SerialName
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
import kotlinx.serialization.json.encodeToStream
import ui.Capture.Companion.getCaptureDuration
import ui.screens.ProfileDetails
import ui.screens.ProfilesManager
import ui.screens.Settings
import java.io.FileInputStream
import java.io.FileOutputStream
import java.nio.file.Path
import java.nio.file.Paths
import kotlin.io.path.absolutePathString
class Profiles {
companion object {
fun getCurrentProfile() = load(Settings.userSettings.getProperty("current_profile", null))
@OptIn(ExperimentalFoundationApi::class)
@Composable
fun currentProfile(modifier: Modifier) { //TODO: store profile in mutableState so component updates when currentProfile
var profile by remember { mutableStateOf(getCurrentProfile()) }
val navigator = LocalNavigator.currentOrThrow
Text(
color = MaterialTheme.colors.onPrimary,
text = if (profile != null) profile!!.name else "Not Selected",
textAlign = TextAlign.Center,
modifier = modifier
.background(MaterialTheme.colors.primary, CircleShape)
.padding(5.dp)
.width(150.dp)
.combinedClickable(
onClick = { if (profile != null) navigator.push(ProfileDetails(profile!!)) },
onDoubleClick = { navigator.push(ProfilesManager()) }
)
)
}
data class Profile(
val name: String,
@SerialName("ai_name")
val aiName: String,
@SerialName("program_name")
val programName: Path,
val commands: List<Command>
)
data class Command(
@SerialName("wording_Variants")
val wordingVariants: List<String>,
val triggers: List<String>
)
@OptIn(ExperimentalSerializationApi::class)
fun load(profile: String?): Profile? {
if (profile != null) {
val stream = FileInputStream(getProfileLocation(profile))
val file = Json.decodeFromStream<Profile>(stream)
stream.close()
return file
}
else {
return null
}
//TODO: Add logging messages
}
@OptIn(ExperimentalSerializationApi::class)
fun save(profile: Profile) {
val stream = FileOutputStream(getProfileLocation(profile.name))
Json.encodeToStream(profile, stream)
stream.flush()
stream.close()
}
fun getProfileLocation(profile: String) = Paths.get("profiles/$profile.profile").absolutePathString()
}
}

59
src/main/kotlin/ui/TopBar.kt
View File

@ -1,59 +0,0 @@
package ui
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.material.Button
import androidx.compose.material.Icon
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ArrowBackIosNew
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
import androidx.compose.ui.unit.dp
import androidx.constraintlayout.compose.ConstraintLayout
import cafe.adriel.voyager.navigator.LocalNavigator
import cafe.adriel.voyager.navigator.currentOrThrow
import ui.screens.Settings
class TopBar {
companion object {
@Composable
fun topBar(screenCount: Int) {
ConstraintLayout(
Modifier
.fillMaxWidth()
) {
val (back, capture, settings) = createRefs()
if (screenCount > 1) {
backButton(Modifier.constrainAs(back) {
top.linkTo(parent.top)
start.linkTo(parent.start, margin = 10.dp)
})
}
Capture.captureContext(Modifier.constrainAs(capture) {
top.linkTo(parent.top)
centerHorizontallyTo(parent)
})
Settings.settingsButton(Modifier.constrainAs(settings) {
top.linkTo(parent.top)
end.linkTo(parent.end, margin = 10.dp)
})
}
}
@Composable
fun backButton(modifier: Modifier) {
val navigator = LocalNavigator.currentOrThrow
Button(
modifier = modifier,
onClick = { navigator.pop()
}) {
Icon(
imageVector = Icons.Filled.ArrowBackIosNew,
contentDescription = "Go Back"
)
}
}
}
}

5
src/main/kotlin/UserSettings.kt → src/main/kotlin/ui/UserSettings.kt
View File

@ -1,3 +1,6 @@
package ui
import logger
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
@ -22,7 +25,7 @@ class UserSettings: Properties() {
fun defaults() {
setProperty("capture_duration", "5")
setProperty("ai_name", "Mia")
setProperty("default_ai_name", "Mia")
}
fun save() {

1
src/main/kotlin/ui/screens/Home.kt
View File

@ -1,6 +1,7 @@
package ui.screens
import androidx.compose.desktop.ui.tooling.preview.Preview
import androidx.compose.material.Text
import androidx.compose.runtime.Composable
import cafe.adriel.voyager.core.screen.Screen
import ui.Capture

15
src/main/kotlin/ui/screens/ProfileDetails.kt Normal file
View File

@ -0,0 +1,15 @@
package ui.screens
import androidx.compose.runtime.Composable
import cafe.adriel.voyager.core.screen.Screen
import ui.Profiles.Companion.Profile
class ProfileDetails(profile: Profile): Screen {
@Composable
override fun Content() {
//TODO: Display all fields for profile
// Allow editing
// Save Profile Button
// Delete Profile Button
}
}

4
src/main/kotlin/ui/screens/Profiles.kt
View File

@ -1,4 +0,0 @@
package ui.screens
class Profiles {
}

14
src/main/kotlin/ui/screens/ProfilesManager.kt Normal file
View File

@ -0,0 +1,14 @@
package ui.screens
import androidx.compose.runtime.Composable
import cafe.adriel.voyager.core.screen.Screen
class ProfilesManager: Screen {
@Composable
override fun Content() {
//TODO: List of profiles in profiles folder onDoubleClick = ProfileDetails for that profile, longClick = open confirmation dialog to activate profile. Show name, programName, and aiName
// Create Profile button
// Delete Profile button
}
}

14
src/main/kotlin/ui/screens/Settings.kt
View File

@ -1,15 +1,19 @@
package ui.screens
import UserSettings
import ui.UserSettings
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.material.Button
import androidx.compose.material.ButtonDefaults
import androidx.compose.material.Icon
import androidx.compose.material.MaterialTheme
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Settings
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.unit.dp
import cafe.adriel.voyager.core.screen.Screen
import cafe.adriel.voyager.navigator.LocalNavigator
@ -34,10 +38,14 @@ class Settings: Screen {
fun settingsButton(modifier: Modifier) {
val navigator = LocalNavigator.currentOrThrow
Button(
shape = CircleShape,
colors = ButtonDefaults.buttonColors(contentColor = MaterialTheme.colors.onPrimary ,backgroundColor = Color.Transparent),
elevation = ButtonDefaults.elevation(0.dp),
modifier = modifier,
onClick = {
navigator.push(Settings())
}) {
}
) {
Icon(
Icons.Filled.Settings,
contentDescription = "Settings"
@ -49,7 +57,7 @@ class Settings: Screen {
fun setting(setting: Map.Entry<Any?, Any?>) {
when(setting.key) {
"capture_duration" -> Capture.captureDurationSlider()
"ai_name" -> null //TODO: TextField
"default_ai_name" -> null //TODO: TextField
else -> null
}
}

11
src/main/resources/logback.xml
View File

@ -1,5 +1,4 @@
<configuration>
<!-- Define the log levels -->
<logger name="capture_logger" level="DEBUG">
<appender-ref ref="CAPTURE_LOGS_FILE" />
<appender-ref ref="STDOUT" />
@ -10,16 +9,16 @@
<appender-ref ref="STDOUT" />
</root>
<!-- Define the specific class appender -->
<!-- Capture appender -->
<appender name="CAPTURE_LOGS_FILE" class="ch.qos.logback.core.FileAppender">
<file>logs/capture.log</file>
<append>true</append>
<append>false</append>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss} - %msg%n</pattern>
<pattern>%d{HH:mm:ss} - %msg%n</pattern>
</encoder>
</appender>
<!-- Define the appender for other logs -->
<!-- Application appended -->
<appender name="APPLICATION_LOGS_FILE" class="ch.qos.logback.core.FileAppender">
<file>logs/application.log</file>
<append>true</append>
@ -28,7 +27,7 @@
</encoder>
</appender>
<!-- Define the STDOUT appender -->
<!-- STDOUT appender -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>