Only support ES5
Add the following <script>
in the <head>
section of index.html
start()
and stop()
should defined in index.js
<script>
function onEvent(eventType) {
if (eventType === 'OnPlayClick') {
JavaScriptInterface.runScript(`start();`);
} else if (eventType === 'OnPauseClick') {
JavaScriptInterface.runScript('stop();');
}
}
function onLog(message) {
console.log(message);
}
</script>
event name |
---|
OnMenuClick |
OnPlayClick |
OnPauseClick |
OnLogClick |
OnSettingClick |
OnCloseClick |
runScript(script)
script
StringrunScriptCallback(script, callback)
script
Stringcallback
StringclickIconButton()
Click the app icon button
on floating widget.
clickPlayButton()
Click the play button
on floating widget.
clickPauseButton()
Click the pause button
on floating widget.
clickLogButton()
Click the log button
on floating widget.
clickSettingButton()
Click the setting button
on floating widget.
clickCloseButton()
Click the close button
on floating widget.
setXY(x, y)
x
Integery
IntegerSet the position of the floating widget.
getX()
Returns Integer
- The x position of the floating widget.
getY()
Returns Integer
- The y position of the floating widget.
showMenu()
Show the menu on floating widget.
hideMenu()
Hide the menu on floating widget.
showPlayButton()
Show the play button
on floating widget.
showPauseButton()
Show the pause button
on floating widget.
getScreenSize()
Returns Object
- {width: Integer, height: Integer}
var sizeObj = getScreenSize();
console.log(sizeObj.width, sizeObj.height);
// 1080 1920
getScreenshot()
Returns Integer
- The image pointer
var img = getScreenshot();
console.log(img);
// 122344533 <- image pointer
releaseImage(img); // Don't forgot release a pointer
getScreenshotModify(cropX, cropY, cropWidth, cropHeight, resizeWidth, resizeHeight, qualitys)
Get screenshot, crop and resize. For speeding up screenshot.
cropX
IntegercropY
IntegercropWidth
IntegercropHeight
IntegerresizeWidth
IntegerresizeHeight
Integerquality
IntegerReturns Integer
- The image pointer
var image = getScreenshotModify(200, 200, 100, 100, 50, 50, 80);
console.log(image); // image width = 50, height = 50
// 12333122
releaseImage(image);
execute(command)
Call exec command in android system. Its permission is same as adb shell
command
StringReturns String
- The result of the execution
var result = execute("ls -al /sdcard");
console.log(result);
// drwxr-xr-x 2 root root 64B 12 14 23:44 Robotmon
tap(x, y, during)
Simulate a tap event after during
milliseconds have passed.
x
Integery
Integerduring
Integer time to wait before tap in millisecondstap(200, 200, 10);
// Will inject a tap down and a tap up event after 10ms to system
tapDown(x, y, during)
x
Integery
Integerduring
Integer time to wait before tapDown in millisecondstapDown(200, 200, 40);
// Will inject a tapDown event after 10ms to system
tapUp(x, y, during)
x
Integery
Integerduring
Integer time to wait before tapUp in millisecondstapUp(200, 200, 40);
// Will inject a tapUo event after 10ms to system
moveTo(x, y, during)
moveTo should be between tapDown
and tapUp
x
Integery
Integerduring
IntegertapDown(500, 300, 40);
moveTo(500, 600, 40);
tapUp(500, 600, 40);
// Will inject a swipe down event
swipe(x1, y1, x2, y2, during)
Simulate a swipe event, using tapDown
, moveTo
and tapUp
event. This function may not work in some game, you should implement yourself.
x1
Integery1
Integerx2
Integery2
Integerduring
Integerswipe(500, 300, 40); // same as above example
// Will inject a swipe down event
keycode(label, during)
Send a key code event to system
Like adb shell
input keyevent command
Android Keycode List
label
Stringduring
Integerkeycode('HOME', 40); // same as keycode('KEYCODE_HOME', 40);
// Will send a HOME event to system
typing(words, during)
Only allow English words
words
Stringduring
Integertyping('Hello!', 100);
// Will type 'H' 'e' 'l' 'l' 'o' '!' 6 words
clone(sourceImg)
Duplicate an image to another.
sourceImg
IntegerReturns Integer
- The image pointer
var oriImage = getScreenshot();
for (var i = 0; i < 10; i++) {
var cloneImage = clone(oriImage);
// modify clone Image here
smooth(cloneImage, 1, 5); // blur
release(cloneImage);
}
release(oriImage);
smooth(sourceImg, smoothType, size)
Same as OpenCV smooth()
function.
sourceImg
IntegersmoothType
Integersize
IntegersmoothType | description |
---|---|
0 | CV_BLUR_NO_SCALE |
1 | CV_BLUR |
2 | CV_GAUSSIAN |
3 | CV_MEDIAN |
4 | CV_BILATERAL |
var img = getScreenshot();
smooth(img, 2, 5); // Gaussian blur
saveImage(img, getStoragePath + '/smooth.png');
releaseImage(img);
convertColor(sourceImg, code)
Same as OpenCV cvtColor()
. Not support different channels. If you want to convert to gray, please use bgrToGray.
Note that getScreenshot
and getScreenshotModify
is BGR order;
sourceImg
Integercode
Integercode | description |
---|---|
40 | CV_BGR2HSV |
52 | CV_BGR2HLS |
See more: OpenCV Types
var img = getScreenshot();
// Convert BGR to HSV color
convertColor(img, 40);
releaseImage(img);
bgrToGray(sourceImg)
Convert form bgr (3 channels) to gray (1 channel).
sourceImg
IntegerReturns Integer
- The gray image pointer
var img = getScreenshot();
var gray = bgrToGray(img); // gray image
releaseImage(img);
releaseImage(gray);
absDiff(sourceImg, targetImg)
Same as OpenCV adbdiff()
.
sourceImg
IntegertargetImg
IntegerReturns Integer
- The image pointer of the difference
var img1 = getScreenshot();
sleep(100);
var img2 = getScreenshot();
var diff = absDiff(img1, img2); // in gray order
releaseImage(img1);
releaseImage(img2);
releaseImage(diff);
threshold(sourceImg, thr, maxThr, code)
Same as OpenCV threshold()
.
sourceImg
Integerthr
FloatmaxThr
Floatcode
Integercode | description |
---|---|
0 | CV_THRES_BINARY |
See more: OpenCV Types
keycode('MENU');
sleep(1000);
var img1 = getScreenshot();
keycode('HOME');
sleep(1000);
var img2 = getScreenshot();
var diff = absDiff(img1, img2); // in gray order
threshold(diff, 100, 255); // set to 0 if <= 100, set to 255 if > 100
var value = getImageColor(diff, 500, 200); // value => {r":255,"g":0,"b":0","a":0}
console.log(value['r']); // current diff value is show on 'r'
// 255
releaseImage(img1);
releaseImage(img2);
releaseImage(diff);
eroid(sourceImg, width, height, x, y)
Same as OpenCV eroid
.
width
, height
, x
, y
is getStructuringElement()
parameters.
sourceImg
Integerwidth
Integerheight
Integerx
Integery
Integervar img = getScreenshot();
threshold(img, 100, 255);
eroid(img, 3, 3, 1, 1);
saveImage(img, getStoragePath() + '/test_eroid.png');
releaseImage(img);
dilate(sourceImg, width, height, x, y)
Same as OpenCV dilate
.
width
, height
, x
, y
is getStructuringElement()
parameters.
sourceImg
Integerwidth
Integerheight
Integerx
Integery
Integervar img = getScreenshot();
threshold(img, 100, 255);
dilate(img, 3, 3, 1, 1);
saveImage(img, getStoragePath() + '/test_dilate.png');
releaseImage(img);
inRange(sourceImg, minB, minG, minR, minA, maxB, maxG, maxR, maxA)
Same as OpenCV inRange + clone + mask
. Filter with range color and clone to new image.
sourceImg
IntegerminB
IntegerminG
IntegerminR
IntegerminA
IntegermaxB
IntegermaxG
IntegermaxR
IntegermaxA
IntegerReturns Integer
- The filtered image pointer
var img = getScreenshot();
var filteredImg = inRange(img, 0, 255, 255, 255, 255, 255, 255, 255); // only keep blue color pixel
saveImage(filteredImg, getStoragePath() + '/test_filterd.png');
releaseImage(img);
releaseImage(filteredImg);
outRange(sourceImg, minB, minG, minR, minA, maxB, maxG, maxR, maxA)
Same as OpenCV inRange + clone + not + mask
. Filter without range color and clone to new image.
sourceImg
IntegerminB
IntegerminG
IntegerminR
IntegerminA
IntegermaxB
IntegermaxG
IntegermaxR
IntegermaxA
IntegerReturns Integer
- The filtered image pointer
var img = getScreenshot();
var filteredImg = outRange(img, 0, 255, 255, 255, 255, 255, 255, 255); // keep all but blue color
saveImage(filteredImg, getStoragePath() + '/test_filterd.png');
releaseImage(img);
releaseImage(filteredImg);
cloneWithMask(sourceImg, mask)
Same as OpenCV copyTo
. Clone image with mask (only support 1 channel)
sourceImg
Integermask
IntegerReturns Integer
- new image pointer with mask
var img1 = getScreenshot();
sleep(100);
var img2 = getScreenshot();
var diff = absDiff(img1, img2);
sleep(100);
var img3 = cloneWithMask(img1, diff);
releaseImage(img1);
releaseImage(img2);
releaseImage(img3);
releaseImage(diff);
houghCircles(sourceImg, method, dp, minDist, p1, p2, minR, maxR)
Same as OpenCV houghCircles
. For finding circles.
sourceImg
Integermethod
Integer (3 = CV_HOUGH_GRADIENT)dp
Float (1) (ratio between input image and input params.)minDist
Float (min distance between circles)p1
Float (canny parameter)p2
Float (canny parameter)minR
Integer (min radius)maxR
Integer (max radius)Returns Object
- Array of circles
var img = getScreenshot();
var points = houghCircles(img, 3, 1, 8, 4, 8, 6, 14);
console.log(points); // {"0": {"x": 102, "y": "233", "r": 9}}
releaseImage(img);
canny(sourceImg, t1, t2, apertureSize)
Same as OpenCV canny
sourceImg
Integert1
Floatt2
FloatapertureSize
IntegerReturns Integer
- The canny image pointer
var img = getScreenshot();
threshold(img, 30, 255);
eroid(img, 5, 5, 1, 1);
var cannyImg = canny(img, 50, 150, 3);
saveImage(cannyImg, getStoragePath() + '/test_canny.png');
releaseImage(img);
releaseImage(cannyImg);
findContours(cannyImgPtr, minArea, maxArea)
Same as OpenCV findContours
.
cannyImgPtr
Integer (Canny image as input)minArea
FloatmaxArea
FloatReturns Object
- {"0": {x: Integer, y: Integer}
var img = getScreenshot();
threshold(img, 30, 255);
eroid(img, 5, 5, 1, 1);
var cannyImg = canny(img, 50, 150, 3);
var results = findContours(cannyImg, 1000, 10000); // area > 100
console.log(JSON.stringify(results));
// {"0":{"x":537,"y":1850},"1":{"x":133,"y":601}}
releaseImage(img);
releaseImage(cannyImg);
drawCircle(sourceImg, x, y, radius, r, g, b, a)
Draw circle in an image.
sourceImg
Integerx
Integery
Integerradius
Integerr
Integerg
Integerb
Integera
Integervar img = getScreenshot();
drawCircle(img, 100, 100, 10, 0, 0, 255, 0); // draw a blue circle
saveImage(img, getStoragePath() + '/test_drawCircle.png');
releaseImage(img);
getIdentityScore(sourceImg, targetImg)
sourceImg
IntegertargetImg
IntegerReturns Float
- The identity score
keycode('MENU');
sleep(1000);
var img1 = getScreenshot();
keycode('HOME');
sleep(1000);
var img2 = getScreenshot();
var score = getIdentityScore(img1, img2);
console.log(score); // 0.6004924774169922
releaseImage(img1);
releaseImage(img2);
cropImage(sourceImg, x, y, width, height)
Crop image.
x
Integery
Integerwidth
Integerheight
IntegerReturns Integer
- The image pointer
var img = getScreenshot();
var cropImg = cropImage(img, 350, 550, 150, 150);
saveImage(cropImg, getStoragePath() + '/test_crop.png');
releaseImage(img);
releaseImage(cropImg);
findImage(sourceImg, targetImg)
Using OpenCV Template Match
to find image.
sourceImg
IntegertargetImg
IntegerReturns Object
- {x: Integer, y: Integer, score: Float}
var img = getScreenshot();
var cropImg = cropImage(img, 350, 550, 150, 150);
var result = findImage(img, cropImg);
console.log(JSON.stringify(result)); // {"score":0.9999997615814209,"x":350,"y":550}
releaseImage(img);
releaseImage(cropImg);
findImages(sourceImg, targetImg, scoreLimit, resultCountLimit, withoutOverlap)
Same as findImage()
, but find multiple times.
sourceImg
IntegertargetImg
IntegerscoreLimit
IntegerresultCountLimit
IntegerwithoutOverlap
BooleanReturns String
- {"0": {"x": Integer, "y": Integer, "score": Float}, "1": {"x": Integer, "y": Integer, "score": Float}}
, Key is String!
var img = getScreenshot();
var cropImg = cropImage(img, 350, 550, 150, 150);
var result = findImages(img, cropImg, 0.95, 3, true);
console.log(JSON.stringify(result)); // {"0":{"score":0.9999997615814209,"x":350,"y":550}}
releaseImage(img);
releaseImage(cropImg);
resizeImage(sourceImg, width, height)
Resize image.
width
Integerheight
IntegerReturns Integer
- The image pointer
var img = getScreenshot();
var resizeImg = resizeImage(img, 108, 192);
saveImage(resizeImg, getStoragePath() + '/test_resize.png');
releaseImage(img);
releaseImage(resizeImg);
releaseImage(imagePointer)
Very Important! You should call this function with all image pointers.
imagePointer
Integervar img = getScreenshot(); // keep in memory
releaseImage(img); // release from memory
getImageColor(sourceImg, x, y)
Get color of point from an image.
sourceImg
Integerx
Integery
IntegerReturns Object
- {r: Integer, g: Integer, b: Integer, a: Integer}
var img = getScreenshot();
var color = getImageColor(img, 100, 100);
console.log(JSON.stringify(color)); // {"a":0,"b":21,"g":36,"r":198}
releaseImage(img);
getImageSize(imgPtr)
imgPtr
IntegerReturns Object
- {width: Integer, height: Integer}
var img = getScreenshot();
var size = getImageSize(img);
console.log(JSON.stringify(size)); // {"height":1920,"width":1080}
releaseImage(img);
saveImage(imgPtr, path)
Save image to disk.
imgPtr
Integerpath
Stringvar img = getScreenshot();
saveImage(img, getStoragePath + '/test_save.png');
releaseImage(img);
openImage(path)
Open image from disk.
path
StringReturns Integer
- The image pointer
var img = openImage(getStoragePath + '/test_save.png');
releaseImage(img);
sleep(milliseconds)
Like sleep
function in C language, pause current process.
milliseconds
Integerconsole.log('Hello');
sleep(1000);
console.log('Andy');
getStoragePath()
Get Robotmon folder. Like /sdcard/Robotmon
.
Returns String
- The storage path
console.log(getStoragePath());
getImageFromURL(url)
Get image from an url.
url
StringReturns Integer
- The image pointer
getImageFromBase64(base64)
Get image from a base64 string.
base64
StringReturns Integer
- The image pointer
getBase64FromImage(imgPtr)
Get base64 string from an image.
imgPtr
IntegerReturns String
- base64
readFile(path)
Read a file as string.
path
StringReturns String
- The text of the file
writeFile(path, text)
Write a string to a file.
path
Stringtext
Stringencrypt(script)
Encrypted a string
script
StringReturns String - The encrypted script
runEncryptedScript(script)
Run an encrypted javascript string.
script
String - The script is encrypted by encrypt
runScript(script)
Run a javascript string.
script
StringhttpClient(method, url, body, headers)
Do a http request.
method
Stringurl
Stringbody
Stringheaders
ObjectReturns String
- The result
httpClient('GET', 'http://httpbin.org/get', '', {});
httpClient('POST', 'http://httpbin.org/post', 'body data', {});
httpClient('POST', 'http://httpbin.org/post', 'foo=bar&bar=foo', {'Content-Type': 'application/x-www-form-urlencoded'});
importJS(library)
Import an JS library.
library
StringimportJS('RBM-0.0.2') // import shared library in libs
importJS('js/customerJS') // import local library
getVirtualButtonHeight()
Returns Integer
- The height of the virtual button
The RBM library is an API wrapper of the Robotmon JavaScript APIs.
property | description |
---|---|
appName | The name of the script. |
oriScreenWidth | The width of developer's phone. |
oriScreenHeight | The height of developer's phone. |
oriVirtualButtonHeight | The virtual button height of developer's phone(getVirtualButtonHeight() ). If no virtual button in app, just set to 0 . |
oriResizeFactor | The resize ratio of the screenshot in developer's environment. For screencrop() . Range from 0 to 1 . |
eventDelay | The delay milliseconds of the event. |
imageThreshold | The threshold of image recognition. Range from 0 to 1 . |
imageQuality | The compression level of the image. Range from 0 to 100 . |
resizeFactor | The resize ratio of the screenshot in user's environment. Same as oriResizeFactor is better. Range from 0 to 1 . |
// Import RBM library
importJS('RBM-0.0.2');
// Initial RBM config
var config = {
appName: 'com.your.script',
oriScreenWidth: 1080,
oriScreenHeight: 1920,
oriVirtualButtonHeight: 0,
oriResizeFactor: 0.6,
eventDelay: 200,
imageThreshold: 0.85,
imageQuality: 80,
resizeFactor: 0.6,
};
// Create RBM instance
var rbm = new RBM(config);
// Important! Calculate the screen size, call it after start pressed!
rbm.init();
// Then using the following APIs of the RBM library
rbm.log(args)
args
any type - if argument is object, it will convert object to JSON stringFor general output of logging information.
rbm.currentApp()
Returns Object
- The current app in foreground. {packageName: String, activityName: String}
rbm.startApp(packageName, activityName)
packageName
StringactivityName
StringLaunch an app by packageName
and activityName
.
rbm.stopApp(packageName)
packageName
StringClose an app by packageName
.
rbm.click(position)
position
Object - {x: Integer, y: Integer}
rbm.tapDown(position)
position
Object - {x: Integer, y: Integer}
rbm.tapUp(position)
position
Object - {x: Integer, y: Integer}
rbm.moveTo(position)
position
Object - {x: Integer, y: Integer}
rbm.swipe(from, to, steps)
from
Object - {x: Integer, y: Integer}
to
Object - {x: Integer, y: Integer}
steps
Integer - Interpolation points between from
and to
rbm.keycode(label)
label
Stringrbm.typing(words)
words
Stringrbm.sleep()
Sleep with eventDelay
.
rbm.getImagePath()
// /sdcard/Robotmon/scripts/com.your.app/images
Returns String
- The path of the image folder. All about images used in this library will load and save within this folder.
rbm.screenshot(filename)
filename
StringSave the screenshot in rbm.getImagePath()
.
rbm.oriScreencrop(filename, fromX, fromY, toX, toY)
// Examples:
rbm.oriScreencrop('startButton.png', 100, 200, 200, 300)
filename
StringfromX
IntegerfromY
IntegertoX
IntegertoY
IntegerCrop the original screenshot and save it with filename
. This function will resize the image with oriResizeFactor
and compress with imageQuality
.
rbm.screencrop(filename, fromX, fromY, toX, toY)
// Examples:
rbm.screencrop('startButton.png', 100, 200, 200, 300)
filename
StringfromX
IntegerfromY
IntegertoX
IntegertoY
IntegerCrop the screenshot and save it with filename
. This function will resize the image with resizeFactor
and compress with imageQuality
.
rbm.findImage(filename, threshold)
// Examples:
rbm.findImage('startButton.png', 0.9)
filename
Stringthreshold
FloatReturns Object
- Find the image with filename
in screen. {x: Integer, y: Integer, score: Float}
rbm.findImages(filename, threshold, countLimit, allowOverlap, deep)
// Examples:
rbm.findImages('startButton.png', 0.9, 3, false, false)
filename
Stringthreshold
FloatcountLimit
IntegerallowOverlap
Booleandeep
BooleanReturns Object
- Find the image with filename
in screen. {x: Integer, y: Integer, score: Float}
rbm.imageExists(filename, threshold)
filename
Stringthreshold
FloatReturns Boolean
- Whether the image exists in screen.
rbm.imageClick(filename, threshold)
filename
Stringthreshold
FloatClick the image if the image exists in screen.
rbm.imageWaitClick(filename, timeout, threshold)
filename
Stringtimeout
Integerthreshold
FloatClick the image if the image exists in screen until timeout (milliseconds).
rbm.imageWaitShow(filename, timeout, threshold)
filename
Stringtimeout
Integerthreshold
FloatBlock until the image is found or timeout
rbm.imageWaitGone(filename, timeout, threshold)
filename
Stringtimeout
Integerthreshold
FloatBlock until the image is gone or timeout
rbm.keepScreenshot()
Keep the screenshot in memory. To avoid too many times screencap.
rbm.screencrop(fromX, fromY, toX, toY)
fromX
IntegerfromY
IntegertoX
IntegertoY
IntegerKeep the partial screenshot in memory. To avoid too many times screencap.
rbm.releaseScreenshot()
Release the screenshot in memory.
// Screencap three times
rbm.imageClick('apple.png', 0.9); // screencap, and release
rbm.imageClick('banana.png', 0.9); // screencap, and release
rbm.imageClick('cat.png', 0.9); // screencap, and release
// Screencap only one time (used when the screen has not changed)
rbm.keepScreenshot(); // screencap
rbm.imageClick('apple.png', 0.9); // no screencap, no release
rbm.imageClick('banana.png', 0.9); // no screencap, no release
rbm.imageClick('cat.png', 0.9); // no screencap, no release
rbm.releaseScreenshot(); // release
message Empty {}
message Response {
string message = 1;
}
message RequestRunScript {
string script = 1;
}
message RequestScreenshot {
int32 cropX = 1;
int32 cropY = 2;
int32 cropWidth = 3;
int32 cropHeight = 4;
int32 resizeWidth = 5;
int32 resizeHeight = 6;
int32 quality = 7;
}
message RequestTap {
int32 x = 1;
int32 y = 2;
int32 during = 3;
}
message ResponseScreenshot {
bytes image = 1;
}
message ResponseScreenSize {
int32 width = 1;
int32 height = 2;
}
service GrpcService {
rpc RunScript(RequestRunScript) returns (Response) {}
rpc Logs(Empty) returns (stream Response) {}
rpc GetScreenshot(RequestScreenshot) returns (ResponseScreenshot) {}
rpc GetScreenSize(Empty) returns (ResponseScreenSize) {}
rpc Tap(RequestTap) returns (Response) {}
rpc TapDown(RequestTap) returns (Response) {}
rpc TapUp(RequestTap) returns (Response) {}
rpc MoveTo(RequestTap) returns (Response) {}
}