desktop-voice-assistant
Easily extensible voice controlled assistant
Last updated 7 months ago by aktyn .
MIT · Repository · Bugs · Original npm · Tarball · package.json
$ cnpm install desktop-voice-assistant 
SYNC missed versions from official npm registry.

Desktop Voice Assistant

Current Version GitHub license GitHub issues GitHub Stars

Voice controlled assistant for desktop environment. Easily extensible by installing external procedures like any node package.

Install

npm i desktop-voice-assistant

Example usage

const Assistant = require('desktop-voice-assistant');

const open = require('open');//only for matter of this example

//Example procedure which opens google.com page with search results
class MyProcedureSearch extends Assistant.ProcedureBase {//it's not necessary but recommended to extend your class
	/**
	 @param {ResultSchema[]} results
	 */
	constructor(results) {
		super(results);
		
		//PREPARE SOMETHING HERE OR HANDLE IT AND FINISH OR PASS TO UPDATE
		console.log('prepare something');
		
		this.update(results);
	}
	
	/**
	 @param {ResultSchema[]} results
	 */
	update(results) {
		//you can wait for final recognition results to check alternative results
		if( results.some(res => res.type === Assistant.RESULT_TYPE.INTERIM) )
			return;
		
		results.sort((r1, r2) => r2.confidence - r1.confidence);
		
		for (let {result} of results) {//from most confident
			let match = result.match(/search (.*)/i);
				
			if (match) {//found most confident final result
				
				Assistant.classifyDesktopContent().then(desktop_content => {
				    // YOU CAN DECIDE WHAT TO DO DEPENDING ON YOUR CURRENT DESKTOP CONTENT
					 
					if( desktop_content.WEBSITE > 0.5 || desktop_content.TEXT > 0.5 )//if you are in text environment
						this.searchForText( match[1] );
					else
						this.searchInGoogle( match[1] );
				 }).catch(console.error);
				
				break;
			}
		}
		
		this.finished = true;
		
		// IF YOU WANT TO SHOW NOTIFICATION OR PRINT ANSWER IN LISTENER WINDOW - YOU CAN DO IT LIKE THAT
		/*******************************
			this.notification = {
				content: equation
			};
			this.answer = {
				message: equation,
				loud: true
			};
		*******************************/
		
		// FOR MULTILINGUAL SUPPORT YOU CAN CHECK CHOSEN LANGUAGE WITH THIS FUNCTION
		//Assistant.getLang();
	}
	
	/** @param {string} text */
	searchForText(text) {
		//press ctrl + f remotely
		Assistant.Robot.tapKey('f', 'control');
		
		setTimeout(() => Assistant.Robot.typeString(text));
	}
	
	/** @param {string} text */
	searchInGoogle(text) {
		open('https://google.com/search?q=' + encodeURI(text.trim())).catch(console.error);
	}
}
//you say anything that matches regexp of chosen language to trigger this procedure
MyProcedureSearch.regexp = {
	'en-US': /^search .+/i
};
/***********************************************************************************/

//procedures can be installed like any node package
Assistant.init([...Assistant.procedures, MyProcedureSearch], {
	open_listener: true,
	ws_port: 3456,//ws_port for websocket connection
	express_port: 4567,//required for browser notifications to work or for listening from other location
	//chrome_command: 'start chrome',//or absolute path to google chrome executable
	use_native_notifications: false,//if false - notifications will be handled by browser
	//lang: 'pl-PL'   //for multilingual procedures you can specify which language to recognize
					// check Assistant.langCodes for list of available webkit speech recognition codes
}).catch(console.error);

Usage information

  • Browser notifications works only in secure web context so if you are not mean to use this package locally - make sure to connect through https.
  • Function "classifyDesktopContent" takes desktop screenshot and uses tensorflow.js to classify it's content into several categories like Website, Game, App. You can use it to differently interpret commands as shown in example above.
    NOTE: convolutional neural network is not trained well yet. I am still extending dataset of screenshots for further training. Any help will be appreciated ;)

Current Tags

  • 1.1.3                                ...           latest (7 months ago)

7 Versions

  • 1.1.3                                ...           7 months ago
  • 1.1.2                                ...           7 months ago
  • 1.1.1                                ...           7 months ago
  • 1.1.0                                ...           7 months ago
  • 1.0.2                                ...           7 months ago
  • 1.0.1                                ...           7 months ago
  • 1.0.0                                ...           7 months ago
Maintainers (1)
Downloads
Today 0
This Week 0
This Month 0
Last Day 0
Last Week 7
Last Month 7
Dependencies (17)
Dev Dependencies (2)
Dependents (0)
None

Copyright 2014 - 2017 © taobao.org |