We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
VoiceAssistant/src/mainwindow.cpp
Line 880 in d7d2f8e
#include "mainwindow.h" #include "global.h" #include "jokes.h" #include "modeldownloader.h" #include "plugins/bridge.h" #include "recognizer.h" #include "ui_mainwindow.h" #include "utils.h" #include <QCloseEvent> #include <QDebug> #include <QDialogButtonBox> #include <QDir> #include <QJsonArray> #include <QJsonDocument> #include <QJsonObject> #include <QLibrary> #include <QLineEdit> #include <QMediaPlayer> #include <QMessageBox> #include <QPluginLoader> #include <QProcess> #include <QRandomGenerator> #include <QSaveFile> #include <QSystemTrayIcon> #include <QTextToSpeech> #include <QThreadPool> #include <QTime> #include <QTimer> #ifdef QT6 #include <QAudioOutput> #endif #include <chrono> using namespace std::chrono_literals; using namespace utils::literals; SpeechToText *recognizer = nullptr; QTextToSpeech *engine = nullptr; QThread *engineThread = nullptr; struct Plugin { PluginInterface *interface = nullptr; QObject *object = nullptr; }; QList<Plugin> plugins; MainWindow *_instance = nullptr; float volume = 1.0; void MainWindow::Action::run(const QString &text) const { if (!funcName.isEmpty()) { if (MainWindow::staticMetaObject.indexOfMethod(QString(funcName + L1("()")).toUtf8()) == -1) QMetaObject::invokeMethod(instance(), funcName.toUtf8(), Qt::QueuedConnection, Q_ARG(QString, text)); else QMetaObject::invokeMethod(instance(), funcName.toUtf8(), Qt::QueuedConnection); } if (!responses.isEmpty()) { int randomIndex = (int) QRandomGenerator::global()->bounded(responses.size()); say(responses.at(randomIndex)); } if (!program.isEmpty()) { QProcess p(instance()); p.setProgram(program); p.setArguments(args); p.startDetached(); } if (!sound.isEmpty()) instance()->playSound(sound); } MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent) , ui(new Ui::MainWindow) , player(new QMediaPlayer(this)) , timeTimer(new QTimer(this)) , jokes(new Jokes(this)) , bridge(new PluginBridge(this)) { // Set up UI ui->setupUi(this); ui->content->setFocus(); // Set instance for instance() _instance = this; // Set audio output device #ifdef QT6 audioOutput = new QAudioOutput(this); player->setAudioOutput(audioOutput); #endif // Set up recognizer recognizer = new SpeechToText(this); connect(recognizer, &SpeechToText::stateChanged, this, &MainWindow::onSTTStateChanged); connect(recognizer, &SpeechToText::languageChanged, this, &MainWindow::loadCommands, Qt::QueuedConnection); #if !NEED_MICROPHONE_PERMISSION recognizer->setup(); #endif // Set up text to speech engineThread = new QThread(this); connect(engineThread, &QThread::finished, engineThread, &QObject::deleteLater); threading::runFunctionInThreadPool(&MainWindow::setupTextToSpeech); // Connect the actions connect(ui->actionAbout_Qt, &QAction::triggered, qApp, &QApplication::aboutQt); connect(ui->actionHas_word, &QAction::triggered, this, &MainWindow::onHasWord); connect(ui->actionOpen_downloader, &QAction::triggered, this, &MainWindow::openModelDownloader); connect(ui->action_About, &QAction::triggered, this, &MainWindow::onHelpAbout); // Set up time timer timeTimer->setInterval(1s); connect(timeTimer, &QTimer::timeout, this, &MainWindow::updateTime); timeTimer->start(); updateTime(); connect(recognizer->device(), &Listener::doneListening, this, &MainWindow::doneListening); connect(recognizer->device(), &Listener::textUpdated, this, &MainWindow::updateText); connect(recognizer->device(), &Listener::wakeWord, this, &MainWindow::onWakeWord); connect(ui->action_Quit, &QAction::triggered, this, &QWidget::close); connect(ui->muteButton, &QCheckBox::clicked, this, &MainWindow::mute); setupTrayIcon(); loadPlugins(); QTimer::singleShot(3s, jokes, &Jokes::setup); } void MainWindow::addCommand(const Action &a) { instance()->commands.append(a); instance()->saveCommands(); } void MainWindow::playSound(const QString &_url) { QUrl url(_url); if (QFile::exists(_url)) url = QUrl::fromLocalFile(_url); #ifdef QT6 player->setSource(url); #else player->setMedia(url); #endif player->play(); } MainWindow *MainWindow::instance() { return _instance; } void MainWindow::onHelpAbout() { QMessageBox::about( this, tr("About VoiceAssistant"), tr("<h1>Voice Assistant</h1>\n" "<p>A resource-efficient and customizable voice assistant written in c++.</p>\n" "<h3>About</h3>\n" "<table border=\"0\" style=\"border-collapse: collapse; width: 100%;\">\n" "<tbody>\n" "<tr>\n" "<td style=\"text-align: right; padding-right: 5px;\">Version:</td>\n" "<td style=\"text-align: left; padding-left: 5px;\">%1</td>\n" "</tr>\n" "<tr>\n" "<td style=\"text-align: right; padding-right: 5px;\">Qt Version:</td>\n" "<td style=\"text-align: left; padding-left: 5px;\">%2</td>\n" "</tr>\n" "<tr>\n" "<td style=\"text-align: right; padding-right: 5px;\">Homepage:</td>\n" "<td style=\"text-align: left; padding-left: 5px;\"><a " "href=\"https://github.com/tim-gromeyer/VoiceAssistant\">https://github.com/" "tim-gromeyer/VoiceAssistant</a></td>\n" "</tr>\n" "</tbody>\n" "</table>\n" "<h3>Credits</h3>\n" "<p>This project uses <a href=\"https://github.com/alphacep/vosk-api\">Vosk</a> which " "is licensed under the <a " "href=\"https://github.com/alphacep/vosk-api/blob/master/COPYING\">Apache License " "2.0</a>.</p>\n" "<p>This project also uses <a href=\"https://github.com/Sygmei/11Zip\" " "target=\"_blank\" rel=\"noopener\">11Zip</a> to unpack the downloaded voice " "modells.</p>") .arg(STR("beta"), qVersion())); } void MainWindow::mute(bool mute) { m_muted = mute; ui->content->setFocus(); if (!recognizer) return; if (mute) { ui->muteButton->setText(tr("Unmute")); ui->muteButton->setToolTip(tr("Unmute")); ui->muteButton->setIcon(QIcon::fromTheme(STR("audio-volume-muted"))); recognizer->pause(); } else { ui->muteButton->setText(tr("Mute")); ui->muteButton->setToolTip(tr("Mute")); ui->muteButton->setIcon(QIcon::fromTheme(STR("audio-input-microphone"))); recognizer->resume(); } muteAction->setText(ui->muteButton->text()); muteAction->setToolTip(ui->muteButton->toolTip()); muteAction->setIcon(ui->muteButton->icon()); muteAction->setChecked(mute); ui->muteButton->setChecked(mute); Q_EMIT muted(); } void MainWindow::toggleMute() { mute(!muteAction->isChecked()); } void MainWindow::toggleVisibilty() { setVisible(!isVisible()); } void MainWindow::closeEvent(QCloseEvent *e) { auto ret = QMessageBox::question(this, tr("Quit?"), tr("Do you really want to quit?")); if (ret != QMessageBox::Yes) e->ignore(); } void MainWindow::setupTextToSpeech() { // Use QThread here because otherwise the stateChanged signal doesn't get emitted qDebug() << "[debug] TTS: Setup QTextToSpeech"; engine = new QTextToSpeech(); engine->moveToThread(engineThread); engineThread->start(); engine->setLocale(QLocale::system()); // WARNING: This might fail! connect(engine, &QTextToSpeech::stateChanged, _instance, &MainWindow::onTTSStateChanged); qDebug() << "[debug] TTS: Setup finished"; } void MainWindow::setupTrayIcon() { trayIcon.reset(new QSystemTrayIcon(QGuiApplication::windowIcon(), this)); connect(trayIcon.get(), &QSystemTrayIcon::activated, this, [this] { setVisible(!isVisible()); }); muteAction = new QAction(this); muteAction->setCheckable(true); muteAction->setText(tr("Mute")); muteAction->setToolTip(tr("Mute")); muteAction->setIcon(QIcon::fromTheme(STR("audio-input-microphone"))); connect(muteAction, &QAction::triggered, this, &MainWindow::mute); auto *menu = new QMenu(this); menu->addAction(ui->action_Quit); menu->addSeparator(); menu->addAction(muteAction); trayIcon->setContextMenu(menu); trayIcon->setToolTip(qApp->applicationName()); trayIcon->show(); } void MainWindow::onTTSStateChanged() { if (!engine) return; QTextToSpeech::State s = engine->state(); switch (s) { case QTextToSpeech::Speaking: if (player) #ifdef QT6 audioOutput->setVolume(0.3F * volume); #else player->setVolume(int(30 * volume)); #endif break; case QTextToSpeech::Ready: if (player) #ifdef QT6 audioOutput->setVolume(1 * volume); #else player->setVolume(int(100 * volume)); #endif break; default: break; } } void MainWindow::onSTTStateChanged() { switch (recognizer->state()) { case SpeechToText::Running: ui->statusLabel->setText(tr("Waiting for wake word")); break; case SpeechToText::NoModelFound: case SpeechToText::ModelsMissing: ui->statusLabel->setText(recognizer->errorString()); openModelDownloader(); break; default: ui->statusLabel->setText(recognizer->errorString()); } } void MainWindow::updateTime() { ui->timeLabel->setText(QLocale::system().toString(QTime::currentTime())); } void MainWindow::updateText(const QString &text) { ui->textLabel->setText(text); } void MainWindow::onWakeWord() { ui->statusLabel->setText(tr("Listening ...")); if (player) #ifdef QT6 audioOutput->setVolume(0.3F * volume); #else player->setVolume(int(30 * volume)); #endif engine->setVolume(.3 * volume); } void MainWindow::doneListening() { ui->statusLabel->setText(tr("Waiting for wake word")); const QString text = ui->textLabel->text(); ui->textLabel->setText({}); QMetaObject::invokeMethod(this, "processText", Qt::QueuedConnection, Q_ARG(QString, text)); if (player) #ifdef QT6 audioOutput->setVolume(1 * volume); #else player->setVolume(int(100 * volume)); #endif engine->setVolume(1.0F * volume); } void MainWindow::onHasWord() { QDialog dia(this); dia.setWindowTitle(tr("Contains word?")); auto *layout = new QVBoxLayout(&dia); auto *infoLabel = new QLabel(tr("Check if the current language model can (not does) recognize a word."), &dia); infoLabel->setWordWrap(true); layout->addWidget(infoLabel); auto *wordLabel = new QLabel(tr("Word:"), &dia); layout->addWidget(wordLabel); auto *wordEdit = new QLineEdit(&dia); wordEdit->setPlaceholderText(tr("Enter word")); layout->addWidget(wordEdit); auto *buttonBox = new QDialogButtonBox(QDialogButtonBox::Ok, &dia); layout->addWidget(buttonBox); connect(wordEdit, &QLineEdit::textChanged, wordEdit, [wordEdit](const QString &w) { auto word = w.trimmed().toLower(); if (word.isEmpty()) { wordEdit->setStyleSheet(QString()); return; } bool hasWord = SpeechToText::hasWord(word); if (hasWord) wordEdit->setStyleSheet(STR("color: green")); else wordEdit->setStyleSheet(STR("color: red")); }); connect(buttonBox, &QDialogButtonBox::accepted, &dia, &QDialog::accept); dia.exec(); dia.deleteLater(); layout->deleteLater(); infoLabel->deleteLater(); wordLabel->deleteLater(); wordEdit->deleteLater(); buttonBox->deleteLater(); delete layout; delete infoLabel; delete wordLabel; delete wordEdit; delete buttonBox; } void MainWindow::openModelDownloader() { ModelDownloader dia(this); connect(&dia, &ModelDownloader::modelDownloaded, recognizer, &SpeechToText::setup, Qt::QueuedConnection); dia.exec(); } void MainWindow::processText(const QString &text) { if (text.isEmpty()) return; for (const auto &action : qAsConst(commands)) { for (const auto &command : action.commands) { if (text.startsWith(command)) { action.run(text.mid(command.length() + 1)); return; } } } for (const Plugin &plugin : qAsConst(plugins)) { if (!plugin.interface->isValid(text)) continue; plugin.interface->run(text); return; } say(tr("I have not understood this!")); } void MainWindow::loadCommands() { commands.clear(); const QString dir = dir::baseDir() + STR("/commands/") + recognizer->language(); QFile jsonFile(dir + STR("/default.json")); // open the JSON file if (!jsonFile.open(QIODevice::ReadOnly)) { qCritical() << STR("Failed to open %1:\n%2") .arg(jsonFile.fileName(), jsonFile.errorString()) .toStdString() .c_str(); return; } // read all the data from the JSON file QByteArray jsonData = jsonFile.readAll(); jsonFile.close(); // In case of an error QJsonParseError error{}; // create a QJsonDocument from the JSON data QJsonDocument jsonDoc = QJsonDocument::fromJson(jsonData, &error); // Error handling if (error.error != QJsonParseError::NoError) { qCritical() << STR("JSON parsing error at %1: %2") .arg(QString::number(error.offset), error.errorString()); return; } // get the array from the JSON document QJsonArray jsonArray = jsonDoc.array(); for (const auto &value : jsonArray) { // convert the array element to an object QJsonObject jsonObject = value.toObject(); // get the commands array from the JSON object QJsonArray commandsArray = jsonObject[STR("commands")].toArray(); if (commandsArray.isEmpty()) { qWarning() << "No commands specified for:" << jsonObject; continue; } // Command template Action c; // get the function name from the JSON object(optional) c.funcName = jsonObject[STR("funcName")].toString(); // get the responses(optional) QJsonArray responseArray = jsonObject[STR("responses")].toArray(); for (const auto &response : responseArray) c.responses.append(response.toString()); // sound/song to play(optional) c.sound = jsonObject[STR("sound")].toString(); // used to execute external programs(optional) c.program = jsonObject[STR("program")].toString(); QJsonArray args = jsonObject[STR("args")].toArray(); c.args.reserve(args.size()); for (const auto &arg : args) c.args.append(arg.toString()); // reserve memory c.commands.reserve(commandsArray.size()); for (const auto &command : commandsArray) c.commands << command.toString(); commands.append(c); } } void MainWindow::saveCommands() { const QString dir = dir::baseDir() + STR("/commands/") + recognizer->language(); QJsonArray jsonArray; for (const auto &c : qAsConst(commands)) { QJsonObject jsonObject; if (!c.funcName.isEmpty()) jsonObject[STR("funcName")] = c.funcName; if (!c.program.isEmpty()) jsonObject[STR("program")] = c.program; if (!c.sound.isEmpty()) jsonObject[STR("sound")] = c.sound; if (!c.commands.isEmpty()) { QJsonArray commandsArray; for (const auto &command : c.commands) commandsArray.append(command); jsonObject[STR("commands")] = commandsArray; } if (!c.responses.isEmpty()) { QJsonArray responseArray; for (const auto &response : c.responses) responseArray.append(response); jsonObject[STR("responses")] = responseArray; } if (!c.args.isEmpty()) { QJsonArray argsArray; for (const auto &arg : c.args) argsArray.append(arg); jsonObject[STR("args")] = argsArray; } if (!jsonObject.isEmpty()) jsonArray.append(jsonObject); } QJsonDocument jsonDoc(jsonArray); QSaveFile jsonFile(dir + STR("/default.json")); if (!jsonFile.open(QIODevice::WriteOnly)) { qDebug() << STR("Failed to open %1\n%2").arg(jsonFile.fileName(), jsonFile.errorString()); return; } jsonFile.write(jsonDoc.toJson()); if (!jsonFile.commit()) QMessageBox::warning( instance(), tr("Failed to save file"), tr("Failed to write <em>%1</em>.\n%2\nCopy following text and save it manually:\n%3") .arg(jsonFile.fileName(), jsonFile.errorString(), jsonDoc.toJson())); } void MainWindow::loadPlugins() { const auto staticInstances = QPluginLoader::staticInstances(); for (QObject *pluginObject : staticInstances) { auto *interface = qobject_cast<PluginInterface *>(pluginObject); if (!interface) continue; interface->setBridge(bridge); Plugin plugin; plugin.interface = interface; plugin.object = pluginObject; plugins.append(plugin); } QDir pluginsDir; pluginsDir.setPath(dir::pluginDir()); const auto entryList = pluginsDir.entryList(QDir::Files); for (const QString &fileName : entryList) { if (!QLibrary::isLibrary(fileName)) continue; QPluginLoader loader(pluginsDir.absoluteFilePath(fileName)); if (!loader.load()) { qWarning() << "Failed to load plugin" << fileName << "due to following reason:" << loader.errorString() << "\nUse `qtplugininfo` for a more advanced error message!"; } QObject *pluginObject = loader.instance(); if (!pluginObject) continue; pluginObject->setParent(this); PluginInterface *interface = qobject_cast<PluginInterface *>(pluginObject); if (!interface) continue; interface->setBridge(bridge); Plugin plugin; plugin.interface = interface; plugin.object = pluginObject; qInfo() << "Loaded plugin:" << loader.fileName(); plugins.append(plugin); } if (plugins.isEmpty()) return; connect(bridge, &PluginBridge::_say, this, &MainWindow::say, Qt::QueuedConnection); connect(bridge, &PluginBridge::_sayAndWait, this, &MainWindow::bridgeSayAndWait, Qt::QueuedConnection); connect(bridge, &PluginBridge::_ask, this, &MainWindow::bridgeAsk, Qt::QueuedConnection); connect(bridge, &PluginBridge::useWidget, ui->content, &QScrollArea::setWidget, Qt::QueuedConnection); } void MainWindow::say(const QString &text) { if (!engine) { qWarning() << "Can not say following text:" << text << "\nTextToSpeech not set up!"; return; } engine->say(text); } void MainWindow::sayAndWait(const QString &text) { if (!engine) return; QEventLoop loop(instance()); connect(engine, &QTextToSpeech::stateChanged, &loop, [&loop](QTextToSpeech::State s) { if (s != QTextToSpeech::Speaking) loop.quit(); }); say(text); loop.exec(); } QString MainWindow::ask(const QString &text) { if (instance()->m_muted) return {QLatin1String()}; sayAndWait(text); // If this is the case something is definitely wrong Q_ASSERT(recognizer->state() != SpeechToText::Running || recognizer->state() != SpeechToText::Paused); // By making it a pointer we prevent out of scope warnings static QString answer; answer.clear(); QEventLoop loop; recognizer->reset(); connect(recognizer, &SpeechToText::answerReady, &loop, [&loop](const QString &asw) { answer = asw; }); connect(recognizer, &SpeechToText::answerReady, &loop, &QEventLoop::quit); if (instance()->m_muted) return {QLatin1String()}; SpeechToText::ask(); instance()->ui->statusLabel->setText(tr("Waiting for answer...")); loop.exec(); instance()->ui->statusLabel->setText(tr("Waiting for wake word")); instance()->ui->textLabel->setText({}); return answer; } void MainWindow::bridgeSayAndWait(const QString &text) { sayAndWait(text); bridge->pause = false; } void MainWindow::bridgeAsk(const QString &text) { if (!m_muted) recognizer->resume(); bridge->answer = ask(text); bridge->pause = false; } void MainWindow::stop() { if (engine) engine->stop(); if (instance()->player) instance()->player->stop(); } void MainWindow::pause() { if (instance()->player) instance()->player->pause(); } void MainWindow::resume() { if (instance()->player) instance()->player->play(); } void MainWindow::quit() { const QString answer = ask(tr("Are you sure?")); if (answer.startsWith(tr("yes"))) { sayAndWait(tr("Okay")); qApp->quit(); } } void MainWindow::sayTime() { // Get the current time QTime currentTime = QTime::currentTime(); // Time as string without seconds const QString time = QLocale::system().toString(currentTime, QLocale::ShortFormat); say(tr("It is %1").arg(time)); } void MainWindow::applyVolume() { qDebug() << "[debug] Change volume to:" << volume; #ifdef QT6 instance()->audioOutput->setVolume(1.0F * volume); #else if (instance()->player) instance()->player->setVolume(int(100 * volume)); #endif if (engine) engine->setVolume(1.0 * volume); } void MainWindow::volumeUp() { if (volume != 1) volume += 0.1; applyVolume(); } void MainWindow::volumeDown() { if (volume != 0) volume -= 0.1; applyVolume(); } void MainWindow::setVolume(const QString &text) { int volumeInt = utils::wordToNumber(text); volume = (float) volumeInt / 10.0F; applyVolume(); } void MainWindow::tellJoke() { jokes->tellJoke(); } void MainWindow::restart() { const QString answer = ask(tr("Are you sure?")); if (!answer.startsWith(tr("yes"))) return; sayAndWait(tr("Okay")); QStringList args = qApp->arguments(); args.removeFirst(); QProcess::startDetached(qApp->applicationFilePath(), args); qApp->quit(); } MainWindow::~MainWindow() { timeTimer->stop(); timeTimer->deleteLater(); delete ui; int activeThreadCount = QThreadPool::globalInstance()->activeThreadCount(); if (activeThreadCount != 0) { qDebug() << "[debug] Waiting for" << activeThreadCount << "threads to finish."; QThreadPool::globalInstance()->waitForDone(); qDebug() << "[debug] All threads ended"; } recognizer->deleteLater(); delete recognizer; bridge->deleteLater(); delete bridge; jokes->deleteLater(); // Prevent deleting the thread while running if (engineThread) { engineThread->quit(); engineThread->wait(); } if (engine) engine->deleteLater(); delete engine; delete engineThread; delete jokes; delete timeTimer; } /* TODO: Let user add commands via GUI, * see: https://doc.qt.io/qt-6/qwizardpage.html, * https://doc.qt.io/qt-6/qwizard.html * https://doc.qt.io/qt-6/qtwidgets-dialogs-classwizard-example.html */ // TODO: Add settings like disabling tray icon, store language and model path and so on // TODO: Add options for controlling text to speech // TODO: Implement weather as a plugin(so it's easier to exclude), see Qt weather example // TODO: The TextToSpeech voice is horrible, change it!
The text was updated successfully, but these errors were encountered:
First work on a weather plugin. See #5
35519c1
tim-gromeyer
No branches or pull requests
VoiceAssistant/src/mainwindow.cpp
Line 880 in d7d2f8e
The text was updated successfully, but these errors were encountered: