#include "opencv2/imgproc/imgproc.hpp" #include "opencv2/highgui/highgui.hpp" #include "opencv2/gpu/gpu.hpp" #include #include #include #include #include String picdir = "E:\\Users\\j\\documents\\visual studio 2010\\Projects\\simcitybot\\simcitybot\\"; // this stuff needs to be globals or else there is a memory leak, I don't know why. something in here doesn't get cleaned up when its scope is gone HDC hwindowDC,hwindowCompatibleDC = NULL; RECT windowsize; int height,width,srcheight,srcwidth; HBITMAP hbwindow = NULL; Mat src; BITMAPINFOHEADER bi; Point prevclickloc; bool newchrome = false; HANDLE fiddlerprocess; long timeoutstart = -1; HWND chromehwnd = NULL; bool chromehwndfound = false; BOOL CALLBACK enumWindowsProc( HWND hwnd, LPARAM lParam ); //bool clicktemplate(Mat kernel, int click = 1,HWND hwnd = chromehwnd); void clickcoins(); void cleaninbox(); HWND executechrome(); void waitforbuildingstoload(); HWND getchromehwnd(); Mat hwnd2mat(HWND); void mousemoveandclick(Point,HWND); void mousemove(Point,HWND); std::string getImgType(int); bool debug = true; bool timeout(int); bool timeout(int milliseconds = 1000){ if(timeoutstart == -1){ //if we are not in the middle of a timer (it = -1) then set the start of the timer. Keep using this value until timeout is reached, at which point we return true and set timeoutstart back to -1 timeoutstart = (long)GetTickCount(); if(debug){ cout << " timer started "; } } if ((long)GetTickCount() - timeoutstart > milliseconds){ if(debug){ cout << " Timed Out. \n"; } // DeleteObject(chromebitmap); // chromebitmap = NULL; //not sure why this gets fucked up sometimes and needs to be created, causing a memory leak. timeoutstart = -1; return true; } return false; } BOOL CALLBACK enumWindowsProc( __in HWND hwnd, __in LPARAM lParam) { if(chromehwndfound) return true; //we already found it - stop looking. (assumes only one window open) HWND* hwndp; string strPCName; TCHAR buffer[50]; GetWindowText(hwnd,buffer,50); size_t size = 50 + 1; // maximum system name length + 1 char *pMBBuffer = (char *)malloc( size ); wcstombs_s(&size, pMBBuffer, (size_t)size,buffer, (size_t)size);// Convert to char* from TCHAR[] strPCName.assign(pMBBuffer); // Now assign the char* to the string, and there you have it!!! :) free(pMBBuffer); TCHAR compare[50] = TEXT("SimCity"); //christ it took me forever to find this at http://stackoverflow.com/questions/4201893/error-c2446-no-conversion-from-const-char-to-tchar?rq=1 if(wcsstr(buffer, (wchar_t*)compare) != NULL){ //cout << "enumWindowsProc Hwnd:" << hwnd << " name: " << strPCName << "\n"; if(debug){ cout << " SimCity found. HWND: " << hwnd << "\n"; } hwndp = (HWND*)lParam; *hwndp = hwnd; chromehwndfound = true; return true; } return true; } bool clicktemplate(Mat kernel, int click = 1,HWND hwnd = chromehwnd){ Mat result1,kernellarge,src,matchedroi,smallresult1,smallsrc ; if(hwnd == NULL || hwnd == 0) hwnd = getchromehwnd(); if(hwnd == NULL || hwnd == 0) return false; bool returnbool = false; Point opposingpoint; double minVal; double maxVal; Point minLoc; Point maxLoc; static Point prevclickloc; //tired of clicking the same coin twice cout << "chromehwnd:" << chromehwnd; src = hwnd2mat(hwnd); cvtColor(src,src,CV_BGRA2BGR); //get rid of alpha chanel for use in matchTemplate if(!kernel.data){ cout << " error - kernel data not found \n"; waitKey(1000); //cin.get(); return false; } if(!src.data){ cout << " error - src data not found \n"; waitKey(1000); //cin.get(); return false; } matchTemplate(src,kernel,result1,1); if(GetAsyncKeyState(VK_ESCAPE)) cin.get(); /// Localizing the best match with minMaxLoc if(!result1.data){ cout << " error - results not found \n"; return false; } minMaxLoc( result1, &minVal, &maxVal, &minLoc, &maxLoc, Mat() ); if(prevclickloc.x == minLoc.x && prevclickloc.y == minLoc.y){ //make sure previous clicked point is not not the same as current - if it is then get 2nd best match cout << " duplicate click location - getting next best \n"; Mat mask = Mat::ones(result1.rows, result1.cols, CV_8UC1); //from http://stackoverflow.com/questions/10258229/is-there-any-direct-function-to-perform-1d-data-interpolation-table-lookup/10260305#_=_ mask.at(minLoc) = 0;// mask out the nearest neighbour and search for the second nearest neighbour minMaxLoc(result1,&minVal, &maxVal, &minLoc, &maxLoc, mask); //apply a mask } prevclickloc = minLoc;//store click location opposingpoint = Point(minLoc.x + kernel.cols,minLoc.y+kernel.rows); // it doesnt seem to hit the exact pixel i need cvtColor(result1,result1,CV_GRAY2BGR); // cout << "1 minVal: " << minVal << " maxval:" << maxVal << " minLoc:" << minLoc.x << "," << minLoc.y << " maxLoc: " << maxLoc.x << "," << maxLoc.y << " oppP:" << opposingpoint.x << "," << opposingpoint.y <<"\n"; matchedroi = src(Rect(minLoc.x,minLoc.y,min(kernel.cols,src.cols),min(kernel.rows,src.rows))); //show what we matched, blown up //this min*() is to catch unknown error resize(matchedroi,matchedroi,Size(120,100),0,0,INTER_AREA); //show how good of a match it was by printing the results of minMaxLoc on screen putText(matchedroi,SSTR(minVal),Point(5,25),1,1,CV_RGB(0,0,250)); if(minVal < .04){ //anything higher means it did not find a good match rectangle(result1,opposingpoint,minLoc,CV_RGB(0,255,0),4); //draw where the computer found it on results rectangle(matchedroi,Point(0,0),Point(matchedroi.cols,matchedroi.rows),CV_RGB(0,250,0),2); //make our little matching boxes green (for success) if(click == 0){ mousemove(Point(minLoc.x + (kernel.cols/2) , minLoc.y + (kernel.rows / 2)),hwnd); //try to click middle of matched template }else if(click == 1){ mousemoveandclick(Point(minLoc.x + (kernel.cols/2) , minLoc.y + (kernel.rows / 2)),hwnd); //try to click middle of matched template }else if(click == 2){ //do not move mouse } returnbool = true; }else{ //draw failures, just out of curiosity rectangle(result1,opposingpoint,minLoc,CV_RGB(240,0,0),4); line(matchedroi,Point(0,0),Point(matchedroi.cols,matchedroi.rows),CV_RGB(240,0,0),1); returnbool = false; } //post processing, so we can see how the computer did resize(kernel,kernellarge,Size(120,100),0,0,INTER_AREA); //show what we are trying to match imshow("kernel",kernellarge); imshow("matched area",matchedroi); resize(result1,smallresult1,Size(600,480),0,0,INTER_AREA); //show what the computer sees when doing matching imshow("result1",smallresult1); resize(src,smallsrc,Size(600,480),0,0,INTER_AREA); //show what the computer sees when doing matching imshow("small src",smallsrc); waitKey(10); //cin.get(); return returnbool; } void closeeverything(){ if(debug){ cout << " Closing everything \n"; } HWND fiddlerhwnd; while(chromehwnd = getchromehwnd()){ //seems to close and reopen too fast, using same window handle? cout << " closing chromehwnd. " << chromehwnd ; SendMessage(chromehwnd,WM_CLOSE,0,0); //MUST CLOSE chrome FIRST to prevent message going through CloseWindow(chromehwnd); // cin.get(); waitKey(3000); } while((fiddlerhwnd = FindWindow(0,L"Fiddler Web Debugger")) != 0){ CloseWindow(fiddlerhwnd); PostMessage(fiddlerhwnd,WM_CLOSE,0,0); SendMessage(fiddlerhwnd,WM_CLOSE,0,0); //sendmessage (vs post) waits for message to be completed. Post just keeps going //lets wait a bit, just in case. I'm getting a memory leak I cnanot find waitKey(3000); if(debug){ cout << " waiting for fiddler to close. "; } if(GetAsyncKeyState(VK_ESCAPE)) cin.get(); } chromehwnd = NULL; fiddlerhwnd = NULL; DeleteDC(hwindowCompatibleDC); DeleteObject(hbwindow); ReleaseDC(chromehwnd,hwindowDC); hwindowDC = NULL; return; } HWND openchrome(){ timeoutstart = -1; while(!(chromehwnd = getchromehwnd()) && !timeout(10000)){ executechrome(); }; return chromehwnd; } HWND executechrome(){ if(!newchrome){ // do not open up multiple simcity social apps. If it hangs you will have to manually restart LPCWSTR action = L"open"; //yes. the L is required. see http://stackoverflow.com/questions/2230758/what-does-lpcwstr-stand-for-and-how-should-it-be-handled-with - another problem that too me forever to figure out LPCWSTR app = L"C:\\Program Files (x86)\\Google\\chrome\\Application\\chrome.exe"; LPCWSTR URL = L"HTTPS://apps.facebook.com/simcitysocial"; SHELLEXECUTEINFO ShExecInfo; ShExecInfo.cbSize = sizeof(SHELLEXECUTEINFO); ShExecInfo.fMask = SEE_MASK_NOCLOSEPROCESS; ShExecInfo.hwnd = NULL; ShExecInfo.lpVerb = action; ShExecInfo.lpFile = app; // this is obatined within deboured CreateProcess. ShExecInfo.lpParameters = URL; ShExecInfo.lpDirectory = NULL; ShExecInfo.nShow = SW_SHOW; ShExecInfo.hInstApp = NULL; if(ShellExecuteEx(&ShExecInfo)){ waitKey(4000); WaitForInputIdle( ShExecInfo.hProcess, 10000 ); //wait for chrome to open and be idle //DOES NOT WORK with console application :( wcout << "\n opening chrome. " << app << " " << URL << "\n"; newchrome = true; Mat refreshchrome = imread(picdir + "refreshchrome.png"); //sometimes flash doesn't load - have to do this clicktemplate(refreshchrome); } } return NULL; } HANDLE executefiddler(){ LPCWSTR action = L"open"; //yes. the L is required. see http://stackoverflow.com/questions/2230758/what-does-lpcwstr-stand-for-and-how-should-it-be-handled-with - another problem that too me forever to figure out LPCWSTR app = L"C:\\Program Files (x86)\\Fiddler2\\Fiddler.exe"; SHELLEXECUTEINFO ShExecInfo = { 0 }; ShExecInfo.cbSize = sizeof(SHELLEXECUTEINFO); ShExecInfo.fMask = SEE_MASK_NOCLOSEPROCESS; //this has to be set in order for hProcess to return something ShExecInfo.hwnd = 0; ShExecInfo.lpVerb = action; ShExecInfo.lpFile = app; ShExecInfo.lpParameters = 0; ShExecInfo.lpDirectory = 0; ShExecInfo.nShow = SW_SHOW; ShExecInfo.hInstApp =0; ShExecInfo.hProcess = 0; if( ShellExecuteEx(&ShExecInfo) ) { WaitForInputIdle( ShExecInfo.hProcess, INFINITE ); wcout << "\n Opening fiddler. " << app << " \n"; return ShExecInfo.hProcess; //I don't need HWND - thats only to interface with OPencv! windows functions I use Processes } return 0; } HWND getchromehwnd(){ HWND chromehwnd = NULL; EnumWindows(&enumWindowsProc ,(LPARAM)&chromehwnd); //which is correct? chromehwndfound = false; //to save cpu return chromehwnd; } Mat hwnd2mat(HWND hwnd){ if(hwnd == GetDesktopWindow()){ srcwidth = GetSystemMetrics(SM_CXMAXTRACK)+8;//getdesktopwindow at getclientrect only gets one monitor. so it will never find fiddler on the 2nd monitor Stupid :( srcheight = GetSystemMetrics(SM_CYMAXTRACK)-8; //stupid microsoft. desktop != desktop }else{ GetClientRect(hwnd, &windowsize);// get the height and width of the screen srcheight = windowsize.bottom-8; srcwidth = windowsize.right+8; } if(srcheight < 0) srcheight = 0; if(srcwidth < 0) srcwidth = 0; height = srcheight; width = srcwidth; //cout << "window height:" << height << " width: " << width << "\n"; if(!src.data || src.rows != height || src.cols != width){ src.create(height,width,CV_8UC4); src.empty(); } //http://www.codeproject.com/Articles/224754/Guide-to-Win32-Memory-DC //if(hwindowDC == NULL){ //try to save some CPU by only doing this stuff once hwindowDC=GetDC(hwnd); hwindowCompatibleDC=CreateCompatibleDC(hwindowDC); SetStretchBltMode(hwindowCompatibleDC,COLORONCOLOR); //} // create a bitmap //if(hbwindow == NULL){ hbwindow = CreateCompatibleBitmap( hwindowDC, width, height); cout << " handle to newly created bitmaap: " << hbwindow << "\n"; //} HGDIOBJ hOldBmp = SelectObject(hwindowCompatibleDC, hbwindow); //copy from hwindowCompatibleDC to hbwindow // SAVE OLD BITMAP // copy from the window device context to the bitmap device context StretchBlt( hwindowCompatibleDC, 0,0, width, height, hwindowDC, 0, 0,srcwidth,srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors ! //set up headers for use in getDIBits bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx bi.biWidth = width; bi.biHeight = -height; //this is the line that makes it draw upside down or not bi.biPlanes = 1; bi.biBitCount = 32; bi.biCompression = BI_RGB; bi.biSizeImage = 0; bi.biXPelsPerMeter = 0; bi.biYPelsPerMeter = 0; bi.biClrUsed = 0; bi.biClrImportant = 0; GetDIBits(hwindowCompatibleDC,hbwindow,0,height,src.data,(BITMAPINFO *)&bi,DIB_RGB_COLORS); // RESTORE OLD BITMAP SelectObject(hwindowCompatibleDC, hOldBmp); DeleteDC(hwindowCompatibleDC); //trying to stop a memory leak :( DeleteObject(hbwindow); // RELEASE WINDOW DC ReleaseDC(hwnd,hwindowDC); return src; } string getImgType(int imgTypeInt){ int numImgTypes = 35; // 7 base types, with five channel options each (none or C1, ..., C4) int enum_ints[] = {CV_8U, CV_8UC1, CV_8UC2, CV_8UC3, CV_8UC4, CV_8S, CV_8SC1, CV_8SC2, CV_8SC3, CV_8SC4, CV_16U, CV_16UC1, CV_16UC2, CV_16UC3, CV_16UC4, CV_16S, CV_16SC1, CV_16SC2, CV_16SC3, CV_16SC4, CV_32S, CV_32SC1, CV_32SC2, CV_32SC3, CV_32SC4, CV_32F, CV_32FC1, CV_32FC2, CV_32FC3, CV_32FC4, CV_64F, CV_64FC1, CV_64FC2, CV_64FC3, CV_64FC4}; string enum_strings[] = {"CV_8U", "CV_8UC1", "CV_8UC2", "CV_8UC3", "CV_8UC4", "CV_8S", "CV_8SC1", "CV_8SC2", "CV_8SC3", "CV_8SC4", "CV_16U", "CV_16UC1", "CV_16UC2", "CV_16UC3", "CV_16UC4", "CV_16S", "CV_16SC1", "CV_16SC2", "CV_16SC3", "CV_16SC4", "CV_32S", "CV_32SC1", "CV_32SC2", "CV_32SC3", "CV_32SC4", "CV_32F" "CV_32FC1", "CV_32FC2", "CV_32FC3", "CV_32FC4", "CV_64F", "CV_64FC1", "CV_64FC2", "CV_64FC3", "CV_64FC4"}; for(int i=0; i //and add dwmapi.lib to the project //then this code: //TRect BoundsR; //DwmGetWindowAttribute( FHwnd, DWMWA_EXTENDED_FRAME_BOUNDS, //&BoundsR, sizeof(BoundsR) ); windowsize.bottom -= 8;//http://stackoverflow.com/questions/3192232/getwindowrect-too-small-on-windows-7/3192264#3192264 windowsize.left += 8;//https://forums.embarcadero.com/thread.jspa?threadID=79864 windowsize.right -= 8; if(windowsize.left < -10 || windowsize.top < -10){ //sometimes window is negative, but should be a smallnegative. cout << "ERROR windowsize is negetive\n"; return; } POINT original; GetCursorPos(&original); int dx = windowsize.left + to.x; int dy = windowsize.top + to.y; SetCursorPos(dx,dy); //its always a few pixels off, why? waitKey(100); //takes a bit for mouse to arrive if(debug){ cout << " setcursorPos to: " << dx << ", " << dy << " click. \n"; } mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0); waitKey(100); mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0); waitKey(100); //SetCursorPos(original.x,original.y); //put it back to where it was, so you can play games in the other monitor :) } void mousemove(Point to,HWND hwnd = GetDesktopWindow()){ RECT windowsize; GetWindowRect(hwnd, &windowsize);// get the corodinates of the screen //cout << " window size: " << windowsize.left << "," << windowsize.top << ", " << windowsize.right << "," << windowsize.bottom << "\n"; if(windowsize.left < -10 || windowsize.top < -10){ //sometimes window is negative, but should be a smallnegative. cout << "ERROR windowsize is negetive\n"; return; } POINT original; GetCursorPos(&original); int dx = windowsize.left + to.x; int dy = windowsize.top + to.y; SetCursorPos(dx,dy); //its always a few pixels off, why? } void clickcoins(){ Mat lightning,lightning2,coin,coin2,X,use,jobcomplete,material1,material2,askfriends,no,quicksend ; if(chromehwnd == NULL || chromehwnd == 0) chromehwnd = getchromehwnd(); cout << " CLicking coins. chromehwnd: " << chromehwnd << "\n"; lightning = imread(picdir + "tinylightning.png"); lightning2 = imread(picdir + "tinylightning2.png"); coin = imread(picdir + "coin.png"); coin2 = imread(picdir + "coin2.png"); material1 = imread(picdir + "material1.png"); material2 = imread(picdir + "material2.png"); coin = imread("coin.png"); coin2 = imread("coin2.png"); askfriends = imread("askfriends.png"); X = imread("X.png"); use = imread("use.png"); jobcomplete = imread("jobcomplete.png"); while(clicktemplate(material1)); while(clicktemplate(material2)); while(clicktemplate(lightning)); while(clicktemplate(lightning2)); while(clicktemplate(coin)); while(clicktemplate(coin2)); while(clicktemplate(jobcomplete)); //the university stuff clicktemplate(no); while(clicktemplate(use)); //keep using those batteries people send clicktemplate(askfriends); //keep using those batteries people send clicktemplate(quicksend); while(clicktemplate(X)); return; } void cleaninbox(){ Mat youvegotmail = imread(picdir + "youvegotmail.png"); Mat inbox = imread(picdir + "inbox.png"); Mat help= imread(picdir + "help.png"); // Mat close = imread(picdir + "close.png"); Mat accept = imread(picdir + "accept.png"); Mat sendback = imread(picdir + "sendback.png"); Mat X = imread(picdir + "X.png"); Mat clock = imread(picdir + "clock.png"); cout << " Cleaning inbox. \n"; if(clicktemplate(youvegotmail) || clicktemplate(inbox)){ cout << " Mailbox found. opening. \n"; waitKey(3000); while(clicktemplate(sendback,0) || clicktemplate(accept,0)|| clicktemplate(help,0)){ for(int i = 0;i < 20; i++){ //click 10 times cuz theres a lot of those stupid simoleans mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0); mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0); waitKey(100); } }; //click buttons as long as it appears if(clicktemplate(X)){; //close inbox cout << " Waiting for inbox packets to send... \n"; waitKey(20000); //wait for packets to send } } } void visitneighbors(){ RNG rng(12345); Mat goodneighbor,badneighbor,scrollright,scrollleft,visit,goodaction,badaction,twinaction,home,travelling,sendgift,sharerewards,ok,neutralneighbor,neutralneighbor2 ,scrollmaxleft,goodaction2,bonus ; goodneighbor = imread(picdir + "goodneighbor.png"); badneighbor = imread(picdir + "badneighbor.png"); scrollright = imread(picdir + "scrollright.png"); scrollleft = imread(picdir + "scrollleft.png"); visit = imread(picdir + "visit.png"); goodaction = imread(picdir + "goodaction.png"); goodaction2 = imread(picdir + "goodaction2.png"); badaction = imread(picdir + "badaction.png"); twinaction = imread(picdir + "twinaction.png"); home = imread(picdir + "home.png"); travelling = imread(picdir + "travelling.png"); sendgift = imread(picdir + "sendgift.png"); sharerewards = imread(picdir + "sharerewards.png"); ok = imread(picdir + "ok.png"); neutralneighbor = imread(picdir + "neutralneighbor.png"); neutralneighbor2 = imread(picdir + "neutralneighbor2.png"); scrollmaxleft = imread(picdir + "scrollmaxleft.png"); bonus = imread(picdir + "bonus.png"); if(clicktemplate(goodneighbor,0) || clicktemplate(neutralneighbor,0) || clicktemplate(neutralneighbor2,0) || clicktemplate(badneighbor,0)){ waitKey(2000); timeoutstart = -1; clicktemplate(visit); waitKey(9000); //wait for travelling? timeoutstart = -1; while (clicktemplate(bonus,2) && !timeout(120000)) { //2 minutes per neighbor, tops, and the bonus sign has to appear //check if the 'first time visit' box appears if(clicktemplate(sharerewards)){ clicktemplate(ok); } //click randomly cout << "Clicking randomly. \n"; mousemoveandclick(Point(rng.uniform(300, 600),rng.uniform(400, 700)),chromehwnd); waitKey(2000); //click the do action //clicktemplate(badaction); clicktemplate(goodaction); //clicktemplate(goodaction2); if(clicktemplate(sendgift)) break; } //go home while(clicktemplate(home)); }else{//no neighbors with energy found. if(!clicktemplate(scrollright)){ clicktemplate(scrollmaxleft); } } return; }