Jesty - Control your PC with Gestures

Jesty - Control your PC with Gestures © GPL3+

Control a PC with a gesture sensor, associating a key or a mouse click to each gesture.

  • 2,730 views
  • 0 comments
  • 6 respects

Components and supplies

Apps and online services

About this project

I had this idea after seeing this video from seeedstudio where they play pacman on a led matrix using gestures.

For the Arduino sketch I edited the paj7620_15gesture example from seeedstudio's library.

When a gesture is detected, it sends a number from 1 to 15 on the serial port.

Change these two values according to your circumstance

#define GES_REACTION_TIME		600
#define GES_QUIT_TIME			1000

GES_QUIT_TIME is the delay between to gestures, and GES_REACTION_TIME is the maximum length of a continous gesture in milliseconds.

On the computer side there is a program that listen to the incomming data on the serial port.

I used jssc library for serial communication in Java.

Once it receive a number on the serial port it executes an action, that can be to press a key on the keyboard or a button on the mouse.

In the github repository, you can find the exported jar file of the program Jesty.jar.

When you run the program for the first time you will see all the gesture disabled, with no action associated.

If you want to associate an action to a gesture, press the button near it. Now just tipe a key or press a mouse button.

All keys are valid except for Esc key. Pressing Esc key will cancel the operation.

To enable a gesture, click on its checkbox.

Now select the serial port from the dropdown and press START button.

If the program is properly listening on the serial port, the button's label will change to "STOP".

To make a quick test, associate some keys to some gesture, open a text editor and should see character appear each time you do make the associated gesture.

You can edit actions enable/disable gestures even if the program is already listening on the serial port.

When you close the program all the setups are stored in a file data.json in the same folder as Jesty.jar.

At the next run, you will find your previous configuration.

Code

Jesty.inoC/C++
#include <Wire.h>
#include "paj7620.h"

#define GES_REACTION_TIME		600
#define GES_QUIT_TIME			1000

void setup()
{
  Serial.begin(115200);
  if (paj7620Init()) while (true);
}

void loop()
{
  uint8_t data = 0, data1 = 0;

  if (!paj7620ReadReg(0x43, 1, &data))
  {
    switch (data)
    {
      case GES_RIGHT_FLAG:
        delay(GES_REACTION_TIME);
        paj7620ReadReg(0x43, 1, &data);
        
        if (data == GES_LEFT_FLAG)Serial.write(6); // "Right-Left"
        
        else if (data == GES_FORWARD_FLAG)
        {
          Serial.write(8); //"Forward"
          delay(GES_QUIT_TIME);
        }
        else if (data == GES_BACKWARD_FLAG)
        {
          Serial.write(7); //"Backward"
          delay(GES_QUIT_TIME);
        }
        else Serial.write(4); //"Right"

        break;
      case GES_LEFT_FLAG:
        delay(GES_REACTION_TIME);
        paj7620ReadReg(0x43, 1, &data);
        if (data == GES_RIGHT_FLAG)Serial.write(5); //"Left-Right"
        
        else if (data == GES_FORWARD_FLAG)
        {
          Serial.write(8); //"Forward"
          delay(GES_QUIT_TIME);
        }
        else if (data == GES_BACKWARD_FLAG)
        {
          Serial.write(7); //"Backward"
          delay(GES_QUIT_TIME);
        }
        else Serial.write(3); //"Left"
        
        break;
      case GES_UP_FLAG:
        delay(GES_REACTION_TIME);
        paj7620ReadReg(0x43, 1, &data);
        
        if (data == GES_DOWN_FLAG)Serial.write(9); //"Up-Down"
        
        else if (data == GES_FORWARD_FLAG)
        {
          Serial.write(8); //"Forward"
          delay(GES_QUIT_TIME);
        }
        else if (data == GES_BACKWARD_FLAG)
        {
          Serial.write(7); //"Backward"
          delay(GES_QUIT_TIME);
        }
        else Serial.write(1); //"Up"
        
        break;
      case GES_DOWN_FLAG:
        delay(GES_REACTION_TIME);
        paj7620ReadReg(0x43, 1, &data);
        if (data == GES_UP_FLAG) Serial.write(10); //"Down-Up"

        else if (data == GES_FORWARD_FLAG)
        {
          Serial.write(8); //"Forward"
          delay(GES_QUIT_TIME);
        }
        else if (data == GES_BACKWARD_FLAG)
        {
          Serial.write(7); //"Backward"
          delay(GES_QUIT_TIME);
        }
        else Serial.write(2); //"Down"
        
        break;
      case GES_FORWARD_FLAG:
        delay(GES_REACTION_TIME);
        paj7620ReadReg(0x43, 1, &data);
        if (data == GES_BACKWARD_FLAG)
        {
          Serial.write(11); //"Forward-Backward"
          delay(GES_QUIT_TIME);
        }
        else
        {
          Serial.write(8); //"Forward"
          delay(GES_QUIT_TIME);
        }
        break;
      case GES_BACKWARD_FLAG:
        delay(GES_REACTION_TIME);
        paj7620ReadReg(0x43, 1, &data);
        if (data == GES_FORWARD_FLAG)
        {
          Serial.write(12); //"Backward-Forward"
          delay(GES_QUIT_TIME);
        }
        else
        {
          Serial.write(7); //"Backward"
          delay(GES_QUIT_TIME);
        }
        break;
      case GES_CLOCKWISE_FLAG:
        Serial.write(13); //"Clockwise"
        break;
      case GES_COUNT_CLOCKWISE_FLAG:
        Serial.write(14); //"anti-clockwise"
        break;
      default:
        paj7620ReadReg(0x44, 1, &data1);
        if (data1 == GES_WAVE_FLAG)
        {
          Serial.write(15); //"wave"
        }
        break;
    }
  }
  delay(100);
}
Jesty.javaJava
package com.mohamedfadiga.jesty;

import java.awt.EventQueue;
import java.awt.Robot;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.InputEvent;
import java.awt.event.KeyEvent;
import java.awt.event.KeyListener;
import java.awt.event.MouseEvent;
import java.awt.event.MouseListener;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.PrintWriter;
import javax.swing.JFrame;
import javax.swing.JComboBox;
import javax.swing.JCheckBox;
import javax.swing.JButton;
import jssc.SerialPort;
import jssc.SerialPortEvent;
import jssc.SerialPortEventListener;
import jssc.SerialPortException;
import jssc.SerialPortList;
import javax.swing.JLabel;
import javax.swing.WindowConstants;
import org.json.JSONArray;
import org.json.JSONObject;


public class Jesty implements SerialPortEventListener
{
	private int gestNum = 15;
	private String[] labels = {"Up", "Down", "Left", "Right", "Left - Right", "Right - Left", "Backward", "Forward", "Up - Down", 
			"Down - Up", "Forward - Backward", "Backward - Forward", "Clockwise", "Anti-clockwise", "Wave"}; //All the available Gesture 
	private int[] keys = new int[gestNum]; //Array containing the key codes or mouse button associated to each gesture 
	private boolean[] status = new boolean[gestNum]; //Array containing the status of each gesture
	private String path; 
	private JFrame frame;
	private SerialPort serialPort;
	private static Jesty jesty;
	private Robot r; //A Robot object for automated actions like typing keys and clicking mouse button 
	private JSONArray jA;
	
	public static void main(String[] args) 
	{   
		EventQueue.invokeLater(new Runnable() 
		{
			public void run() 
			{
				try 
				{
					jesty = new Jesty();
					jesty.frame.setVisible(true);
				} 
				catch (Exception e){e.printStackTrace();}
			}
		});
	}

	public Jesty() {initialize();}

	private void initialize() 
	{
		try 
		{ 
			r = new Robot();
			path =  new File(Jesty.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath())
					.getParentFile().getAbsolutePath(); //Gets the folder where Jesty.jar is located
			//Reads previously stored data
			BufferedReader br;
			br = new BufferedReader(new FileReader(path+"\\data.json"));
			String line; 
			StringBuilder sb = new StringBuilder();
			while ((line = br.readLine()) != null)sb.append(line);
			br.close();
			jA = new JSONArray(sb.toString());
		} 
		catch (Exception e){}  
		
		frame = new JFrame();
		frame.setResizable(false);
		frame.setBounds(100, 100, 320, 620);
		frame.setDefaultCloseOperation(WindowConstants.DO_NOTHING_ON_CLOSE); //Because we want to save data before closing the program
        frame.addWindowListener( new WindowAdapter() 
        {
            @Override
            public void windowClosing(WindowEvent we) //When the program is closing
            {
            	try 
            	{
            		jA = new JSONArray();
            		for(int i=0;i < gestNum;++i) //puts data into a JSON Array
            		{
            			JSONObject  jO = new JSONObject();
            			jO.put("key", keys[i]);
            			jO.put("active", status[i]);
            			jA.put(i, jO);
            		}           		
            		
            		//Writes JSON to a file 
            		PrintWriter writer = new PrintWriter(path + "\\data.json", "UTF-8");
            		writer.print(jA.toString());
            		writer.close();
            	} 
            	catch (Exception e){}
            	System.exit(0);
            }
        });
		
        frame.getContentPane().setLayout(null);
		
		JComboBox<String> comboBox = new JComboBox<String>(SerialPortList.getPortNames());
		comboBox.setBounds(12, 10, 100, 22);
		frame.getContentPane().add(comboBox);

		JButton btnStart = new JButton("START");
		btnStart.setBounds(160, 9, 97, 25);
		btnStart.addActionListener(new ActionListener()
		{
			@Override
			public void actionPerformed(ActionEvent arg0) //When we press btnStart 
			{
				try 
				{
					if((serialPort == null || !serialPort.isOpened())&& (comboBox.getSelectedItem() != null))
					{
						//Creates, opens and setup serial port  
						serialPort =  new SerialPort(comboBox.getSelectedItem().toString());
						serialPort.openPort();
						serialPort.setParams(SerialPort.BAUDRATE_115200,
							SerialPort.DATABITS_8,
					        SerialPort.STOPBITS_1,
					        SerialPort.PARITY_NONE);
						serialPort.addEventListener(jesty, SerialPort.MASK_RXCHAR);
						btnStart.setText("STOP");
					}
					else if((serialPort != null && serialPort.isOpened()))
					{
						btnStart.setText("START");
						serialPort.closePort();
					}
				} 
				catch (SerialPortException e){e.printStackTrace();}
			}
		});
		frame.getContentPane().add(btnStart);		

		KeyListener keyListener = new KeyListener()
		{
			@Override
			public void keyPressed(KeyEvent e)
			{
				JButton button = (JButton)e.getSource(); //Get the button that generated the event
				button.removeKeyListener(this);
				int code =e.getKeyCode(); 
				if(code == 27)return; //if we press "Esc" key							
				button.setText(""+KeyEvent.getKeyText(code)); //Change the button's label to the key associated to this code
				keys[Integer.parseInt(button.getName())] = code;
			}

			@Override
			public void keyReleased(KeyEvent e){}

			@Override
			public void keyTyped(KeyEvent e){}
		};
		
		MouseListener mouseListener = new MouseListener()
		{
			@Override
			public void mouseClicked(MouseEvent e)
			{
				int b = e.getButton(); //Get which mouse button were clicked 
				JButton button = (JButton)e.getSource(); //Get the button that generated the event
				if(button.getKeyListeners().length == 0 && b==1)//If we are not already waiting for a key and we clicked the left button of the mouse
				{
					button.addKeyListener(keyListener); //Wait for a key to associate to  the gesture
				}
				else if(button.getKeyListeners().length == 1) //If we want to associate a mouse click instead of a key 
				{
					keys[Integer.parseInt(button.getName())] = b * -1; //Stores the negative value of the mouse button to distinguish it from key codes
					button.setText("Mouse " + b);
					button.removeKeyListener(keyListener); 
				}
			}
			
			@Override
			public void mouseEntered(MouseEvent e) {}

			@Override
			public void mouseExited(MouseEvent e) {}

			@Override
			public void mousePressed(MouseEvent e){}

			@Override
			public void mouseReleased(MouseEvent e) {}
		};
		
		ActionListener actionListener = new ActionListener()
		{
			@Override
			public void actionPerformed(ActionEvent event) //When we edit a  checkbox
			{
				JCheckBox checkBox = (JCheckBox)event.getSource(); //Gets the checkbox that generated the event
				status[Integer.parseInt(checkBox.getName())] = checkBox.isSelected(); 	
			}
		};
		
		for(int i = 0;i < gestNum;++i)
		{
			try
			{
				JSONObject jO;
				jO = jA.getJSONObject(i);
				keys[i] = jO.getInt("key");
				status[i] = jO.getBoolean("active");
			}
			catch (Exception e) //jA is null, which probably means that data.json does't exists 
			{
				keys[i] = 0;
				status[i] = false; //Disable all the Gestures
			}

			JLabel label = new JLabel(labels[i]);
			label.setBounds(12, 50+35*i, 120, 16);
						
			JButton button = new JButton(keys[i] > -1?KeyEvent.getKeyText(keys[i]):"Mouse "+ keys[i] * -1); //Sets button's label
			button.setBounds(150, 45+35*i, 120, 25);
			button.setName(""+i); //We give a name to each button to know which one generated a event 
			button.addMouseListener(mouseListener);
			
			JCheckBox checkBox = new JCheckBox("");
			checkBox.setBounds(280, 45+35*i, 25, 25);
			checkBox.setName(""+i);
			checkBox.setSelected(status[i]);
			checkBox.addActionListener(actionListener);
			frame.getContentPane().add(label);
			frame.getContentPane().add(button);
			frame.getContentPane().add(checkBox);
		}
	}

	@Override
	public void serialEvent(SerialPortEvent event) 
	{
		try 
		{
			byte b  = serialPort.readBytes(1)[0]; //Reads first byte  available on serial port 
			if(b>0 && b<16 && keys[b-1] !=0) //if we have a valid gesture, there is a valid key o mouse button associated
			{
				if(status[b-1]) //if the gesture is enabled
				{
					if(keys[b-1] > -1) //if we associated a key
					{
						r.keyPress(keys[b-1]);
						r.keyRelease(keys[b-1]);
					}
					else //We associated a mouse button
					{
						int mask = InputEvent.getMaskForButton(keys[b-1] * -1);
						r.mousePress(mask);
						r.mouseRelease(mask);
					}
				}
			}
	    }
	    catch (Exception e){e.printStackTrace();}
	}
}
Jesty
Repository on githug

Comments

Similar projects you might like

Servo Control with TV Remote Control

Project showcase by eldo85

  • 5,633 views
  • 5 comments
  • 17 respects

DAS Access Control System (DASACS)

Project showcase by DasData

  • 4,557 views
  • 1 comment
  • 23 respects

Control an LED with the Remote Control

Project showcase by Nicholas_N

  • 3,192 views
  • 2 comments
  • 9 respects

Control Your Computer With A Remote Control

Project tutorial by Arduino_Scuola

  • 5,942 views
  • 1 comment
  • 8 respects

IR Remote Control for Presentation PC

Project showcase by PiotrSylwesiuk

  • 2,941 views
  • 1 comment
  • 5 respects
Add projectSign up / Login