V0.3.1 Official

ElectronJS/README.md

@@ -59,19 +59,20 @@ optional:
 npm run make
 ```
 
-### For windows x64
+#### For windows x64
 
-依次执行下面两个cmd即可打包
+依次执行下面两个cmd即可打包并发布，无需执行上面的npm命令
 
 ```
 package_win64.cmd
-clean_win64.cmd
+clean_and_release_win64.cmd
 ```
 
-### For Windows x86
-依次执行下面两个cmd即可打包
+#### For Windows x86
+
+依次执行下面两个cmd即可打包并发布，无需执行上面的npm命令
 
 ```
 package_win32.cmd
-clean_win32.cmd
+clean_and_release_win32.cmd
 ```

ElectronJS/src/taskGrid/FlowChart_CN.html

@@ -100,7 +100,7 @@
                     <div v-if='!useLoop'>
 <!--                        <label>url:</label>-->
 <!--                        <input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["url"]'></input>-->
-                        <label>链接（每行一个链接，有多少行链接整个任务流程就会被执行多少次）：</label>
+                        <label>链接池（每行一个链接，有多少行链接整个任务流程就会被执行多少次）：</label>
                         <textarea onkeydown="inputDelete(event)" class="form-control" rows="2" v-model='nowNode["parameters"]["links"]'></textarea>
                     </div>
                     <label>页面加载最长等待时间（秒）：</label>

ElectronJS/src/taskGrid/invokeTask.html

@@ -201,7 +201,7 @@
                 t.forEach(function (item, index) {
                     para[item.name] = item.value;
                 });
-                $.post(app.$data.backEndAddressServiceWrapper + "/invokeTask?id=" + sId, {
+                $.post(app.$data.backEndAddressServiceWrapper + "/invokeTask", {
                     id: this.task.id,
                     paras: JSON.stringify(para)
                 }, function (result) {
@@ -263,7 +263,7 @@
                     t.forEach(function (item, index) {
                         para[item.name] = item.value;
                     });
-                    $.post(app.$data.backEndAddressServiceWrapper + "/invokeTask?id=" + sId, {
+                    $.post(app.$data.backEndAddressServiceWrapper + "/invokeTask", {
                         id: this.task.id,
                         paras: JSON.stringify(para)
                     }, function (result) {

ElectronJS/src/taskGrid/logic.js

@@ -160,6 +160,7 @@ function addParameters(t) {
         t["parameters"]["scrollWaitTime"] = 1; //滚动后等待时间
         t["parameters"]["maxWaitTime"] = 10; //最长等待时间
         t["parameters"]["paras"] = []; //默认参数列表
+        t["parameters"]["wait"] = 2; //点击后等待时间默认2s
         t["parameters"]["beforeJS"] = ""; //执行前执行的js
         t["parameters"]["beforeJSWaitTime"] = 0; //执行前js等待时间
         t["parameters"]["afterJS"] = ""; //执行后执行的js

ElectronJS/src/taskGrid/logic_CN.js

@@ -160,6 +160,7 @@ function addParameters(t) {
         t["parameters"]["scrollWaitTime"] = 1; //滚动后等待时间
         t["parameters"]["maxWaitTime"] = 10; //最长等待时间
         t["parameters"]["paras"] = []; //默认参数列表
+        t["parameters"]["wait"] = 2; //点击后等待时间默认2s
         t["parameters"]["beforeJS"] = ""; //执行前执行的js
         t["parameters"]["beforeJSWaitTime"] = 0; //执行前js等待时间
         t["parameters"]["afterJS"] = ""; //执行后执行的js

ElectronJS/src/taskGrid/taskList.html

@@ -23,6 +23,8 @@
             <h4 style="text-align: center;">{{"Task List~任务列表" | lang}}</h4>
             <p><a v-if="type==3" href="javascript:void(0)" v-on:click="newTask" class="btn btn-primary">{{"New Task~创建新任务" | lang}}</a></p>
             <div v-if="type != 3" style="margin-bottom: 20px">
+                <a class="btn btn-primary" href="https://github.com/NaiboWang/EasySpider/wiki" target="_blank">{{"Software Documentation~软件使用说明文档" | lang}}</a>
+                <a class="btn btn-primary" href="https://github.com/NaiboWang/EasySpider/issues?q=" target="_blank">{{"Ask questions here~官方答疑平台" | lang}}</a>
                 <a class="btn btn-primary" href="https://github.com/NaiboWang/EasySpider/issues/22" target="_blank">{{"See how to run task by schedule~定时执行任务教程" | lang}}</a>
                 <a class="btn btn-primary" href="https://github.com/NaiboWang/EasySpider/wiki/Run-multiple-tasks-in-parallel" target="_blank">{{"See how to run multiple tasks in parallel~同时执行多个任务教程" | lang}}</a>
             </div>

ExecuteStage/easyspider_executestage.py

@@ -105,8 +105,8 @@ class BrowserThread(Thread):
         with open(stealth_path, 'r') as f:
             js = f.read()
             print("Loading stealth.min.js")
-        browser_t.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {'source': js}) # TMALL 反扒
-        wait = WebDriverWait(self.browser, 10)
+        self.browser.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {'source': js}) # TMALL 反扒
+        WebDriverWait(self.browser, 10)
         self.browser.get('about:blank')
         print("id: ", id)
         if saved_file_name != "":
@@ -182,11 +182,11 @@ class BrowserThread(Thread):
         if switch:
             print(text, text2)
 
-    @atexit.register
-    def clean(self):
-        self.saveData(exit=True)
-        self.browser.quit()
-        sys.exit(0)
+    # @atexit.register
+    # def clean(self):
+    #     self.saveData(exit=True)
+    #     self.browser.quit()
+    #     sys.exit(0)
 
     def saveData(self, exit=False):
         if exit == True or len(self.OUTPUT) >= 100: # 每100条保存一次
@@ -202,6 +202,7 @@ class BrowserThread(Thread):
             self.log = ""
 
     def scrollDown(self, para, rt=""):
+        time.sleep(para["scrollWaitTime"]) # 下拉前等待
         scrollType = int(para["scrollType"])
         try:
             if scrollType != 0 and para["scrollCount"] > 0:  # 控制屏幕向下滚动
@@ -213,7 +214,7 @@ class BrowserThread(Thread):
                     elif scrollType == 2:
                         body.send_keys(Keys.END)
                     time.sleep(para["scrollWaitTime"])  # 下拉完等待
-        except TimeoutException:
+        except:
             self.Log('time out after set seconds when scrolling. ')
             self.recordLog('time out after set seconds when scrolling')
             self.browser.execute_script('window.stop()')
@@ -377,10 +378,10 @@ class BrowserThread(Thread):
             self.moveToElement(node["parameters"], loopValue, loopPath, index)
         elif node["option"] == 8:  # 循环
             self.recordLog("loop")
-            self.loopExcute(node, loopValue, loopPath, index)  # 执行循环
+            self.loopExecute(node, loopValue, loopPath, index)  # 执行循环
         elif node["option"] == 9:  # 条件分支
             self.recordLog("judge")
-            self.judgeExcute(node, loopValue, loopPath, index)
+            self.judgeExecute(node, loopValue, loopPath, index)
 
         # 执行完之后进行等待
         if node["option"] != 0:
@@ -392,7 +393,7 @@ class BrowserThread(Thread):
 
 
     # 对判断条件的处理
-    def judgeExcute(self, node, loopElement, clickPath="", index=0):
+    def judgeExecute(self, node, loopElement, clickPath="", index=0):
         executeBranchId = 0  # 要执行的BranchId
         for i in node["sequence"]:
             cnode = self.procedure[i]  # 获得条件分支
@@ -450,7 +451,7 @@ class BrowserThread(Thread):
             self.executeNode(executeBranchId, loopElement, clickPath, index)
 
     # 对循环的处理
-    def loopExcute(self, node, loopValue, clickPath="", index=0):
+    def loopExecute(self, node, loopValue, clickPath="", index=0):
         time.sleep(0.1)  # 第一次执行循环的时候强制等待1秒
         # self.Log("循环执行前等待0.1秒")
         self.Log("Wait 0.1 second before loop")
@@ -638,8 +639,13 @@ class BrowserThread(Thread):
         self.history["handle"] = self.browser.current_window_handle
         if para["useLoop"]:
             url = loopValue
-        else:
+        elif para["url"] != "about:blank":
             url = self.links[self.urlId]
+            # clear output parameters
+            for key in self.outputParameters:
+                self.outputParameters[key] = ""
+        else:
+            url = list(filter(isnull, para["links"].split("\n")))[0]
         try:
             maxWaitTime = int(para["maxWaitTime"])
         except:
@@ -676,9 +682,6 @@ class BrowserThread(Thread):
             except Exception as e:
                 self.Log(e)
                 self.recordLog(str(e))
-        # clear output parameters
-        for key in self.outputParameters:
-            self.outputParameters[key] = ""
 
 
     # 键盘输入事件
@@ -748,9 +751,6 @@ class BrowserThread(Thread):
         except Exception as e:
             self.Log(e)
             self.recordLog(str(e))
-        time.sleep(0.5)  # 点击之后等半秒
-        self.Log("Wait 0.5 second after clicking element")
-        time.sleep(random.uniform(1, 2))  # 生成一个a到b的小数等待时间
         # 点击前对该元素执行一段JavaScript代码
         try:
             if para["afterJS"] != "":
@@ -866,8 +866,8 @@ class BrowserThread(Thread):
                     content = text
                 except Exception as e:
                     content = "OCR Error"
-                    print("To use OCR, You need to install Tesseract-OCR and add it to the environment variable PATH: https://tesseract-ocr.github.io/tessdoc/Installation.html")
-                    print("要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中：https://blog.csdn.net/u010454030/article/details/80515501")
+                    print("To use OCR, You need to install Tesseract-OCR and add it to the environment variable PATH (need to restart EasySpider after you put in PATH): https://tesseract-ocr.github.io/tessdoc/Installation.html")
+                    print("要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中（添加后需重启EasySpider）：https://blog.csdn.net/u010454030/article/details/80515501\nhttps://www.bilibili.com/video/BV1xz4y1b72D/")
             elif p["contentType"] == 9:
                 content = self.execute_code(2, p["JS"], p["JSWaitTime"], element)
             elif p["contentType"] == 10: # 下拉框选中的值
@@ -887,10 +887,6 @@ class BrowserThread(Thread):
 
     # 提取数据事件
     def getData(self, para, loopElement, isInLoop=True, parentPath="", index=0):
-        if not isInLoop and para["wait"] == 0:
-            time.sleep(1)  # 如果提取数据字段不在循环内而且设置的等待时间为0，默认等待1秒
-            self.Log("Wait 1 second before extracting data")
-        # rt = Time("Extract Data")
         for p in para["paras"]:
             content = ""
             if not (p["contentType"] == 5 or p["contentType"] == 6):  # 如果不是页面标题或URL，去找元素

ExecuteStage/easyspider_executestage_single.py

@@ -280,10 +280,10 @@ def executeNode(nodeId, loopValue="", loopPath="", index=0):
         moveToElement(node["parameters"], loopValue, loopPath, index)
     elif node["option"] == 8:  # 循环
         recordLog("loop")
-        loopExcute(node, loopValue, loopPath, index)  # 执行循环
+        loopExecute(node, loopValue, loopPath, index)  # 执行循环
     elif node["option"] == 9:  # 条件分支
         recordLog("judge")
-        judgeExcute(node, loopValue, loopPath, index)
+        judgeExecute(node, loopValue, loopPath, index)
 
     # 执行完之后进行等待
     if node["option"] != 0:
@@ -295,7 +295,7 @@ def executeNode(nodeId, loopValue="", loopPath="", index=0):
 
 
 # 对判断条件的处理
-def judgeExcute(node, loopElement, clickPath="", index=0):
+def judgeExecute(node, loopElement, clickPath="", index=0):
     # rt = Time("IF Condition")
     global bodyText  # 引入bodyText
     executeBranchId = 0  # 要执行的BranchId
@@ -365,7 +365,7 @@ def get_output_code(output):
     return code
 
 # 对循环的处理
-def loopExcute(node, loopValue, clickPath="", index=0):
+def loopExecute(node, loopValue, clickPath="", index=0):
     time.sleep(0.1)  # 第一次执行循环的时候强制等待1秒
     # Log("循环执行前等待0.1秒")
     Log("Wait 0.1 second before loop")

ExecuteStage/generateExecutable_Linux64.sh

@@ -2,4 +2,4 @@ rm -r build
 rm -r dist
 pyinstaller -F --icon=favicon.ico easyspider_executestage.py
 rm ../ElectronJS/chrome_linux64/easyspider_executestage
-mv dist/easyspider_executestage ../ElectronJS/chrome_linux64/easyspider_executestage
+cp dist/easyspider_executestage ../ElectronJS/chrome_linux64/easyspider_executestage

ExecuteStage/generateExecutable_Macos.sh

@@ -3,5 +3,5 @@ rm -r dist
 pyinstaller -F --icon=favicon.ico easyspider_executestage.py
 rm ../Releases/EasySpider_MacOS_all_arch/easyspider_executestage
 rm ../ElectronJS/easyspider_executestage
-mv dist/easyspider_executestage ../Releases/EasySpider_MacOS_all_arch/easyspider_executestage
+cp dist/easyspider_executestage ../Releases/EasySpider_MacOS_all_arch/easyspider_executestage
 # mv dist/easyspider_executestage ../ElectronJS/easyspider_executestage

Extension/manifest_v3/src/content-scripts/toolkit.vue

@@ -14,11 +14,10 @@
             </div>
             <p style="color:black; margin-top: 10px">● 鼠标移动到元素上后，请<strong>右键</strong>点击或者按<strong>F7</strong>键选中页面元素。
             </p>
-            <p style="color:black; margin-top: 10px">● 操作完成后，如点击”确认采集“后任务流程图内没有”提取数据“操作被添加，<strong>重试一次</strong>即可。</p>
             <p style="color:black; margin-top: 10px">● 通过鼠标左键进行点击时，页面也会有反应，但左键点击发生的操作不会被记录在任务流程中；同理，如果想输入文本框但并不想将动作记录，可以鼠标移动到文本框，并按键盘的<strong>F9</strong>进行输入。
             </p>
             <p style="color:black; margin-top: 10px">● 如果不小心左键点选了元素导致页面跳转，直接后退或者切换回标签页即可。</p>
-
+            <p style="color:black; margin-top: 10px">● 操作完成后，如点击”确认采集“后任务流程图内没有”提取数据“操作被添加，<strong>重试一次</strong>即可。</p>
             {{ initial() }}
           </div>
           <div v-if="list.nl.length==1">
@@ -162,9 +161,9 @@
             </div>
             <p style="color:black">● When your mouse moves to the element, please <strong>right-click</strong> your
               mouse button or press <strong>F7</strong> on the keyboard to select it.</p>
-            <p style="color:black; margin-top: 10px">● After the operation is completed, such as if no "Collect Data" operation is added in the task flowchart after clicking "Confirm Collect", just <strong> retry </strong> again.</p>
             <p style="color:black; margin-top: 10px">● When clicked with the left mouse button, the page will also respond, but this click operation will not be recorded in the task flow. Similarly, if you want to input in a text box but do not want the action to be recorded , you can move the mouse to the text box and press <strong>F9</strong> on the keyboard to input.</p>
             <p style="color:black; margin-top: 10px">● If you accidentally left-click on an element and cause the page to jump, simply go back or switch back to the tab.</p>
+            <p style="color:black; margin-top: 10px">● After the operation is completed, such as if no "Collect Data" operation is added in the task flowchart after clicking "Confirm Collect", just <strong> retry </strong> again.</p>
             {{ initial() }}
           </div>
           <div v-if="list.nl.length==1">

Extension/manifest_v3/src/style/toolkit.css

@@ -25,6 +25,7 @@
 
 .realcontent {
     text-align: left;
+    color: black!important;
     padding-top: 10px !important;
     padding-bottom: 80px !important;
     padding-left: 20px !important;

Releases/EasySpider_windows_amd64/config.json

@@ -1 +1 @@
-{"webserver_address":"http://localhost","webserver_port":8074,"user_data_folder":"./user_data2","absolute_user_data_folder":"D:\\Documents\\Projects\\EasySpider\\Releases\\EasySpider_windows_amd64\\user_data2"}
+{"webserver_address":"http://localhost","webserver_port":8074,"user_data_folder":"./user_data","absolute_user_data_folder":"D:\\Documents\\Projects\\EasySpider\\Releases\\EasySpider_windows_amd64\\user_data2"}

Releases/EasySpider_windows_amd64/config2.json

@@ -1 +0,0 @@
-{"webserver_address":"http://localhost","webserver_port":8074,"user_data_folder":"./user_data2","absolute_user_data_folder":"D:\\Documents\\Projects\\EasySpider\\Releases\\EasySpider_windows_amd64\\user_data2"}

Releases/EasySpider_windows_amd64/readme.txt

@@ -1,11 +1,13 @@
-视频教程：https://www.bilibili.com/video/BV1Fk4y1L7xX/
-Video Tutorial: https://youtube.com/playlist?list=PL0kEFEkWrT7mt9MUlEBV2DTo1QsaanUTp
+Official Site: https://github.com/NaiboWang/EasySpider
 
-这个软件绝对不是特洛伊木马/病毒！如果被像 Windows Defender 这样的杀毒软件误认为是病毒，请进行恢复，或者打开“EasySpider.bat”来运行我们的软件。
+Welcome to promote this software to other friends.
 
-The software is totally not trojan/virus! If mistaken by antivirus software such as windows defender as a virus, please recover it, or open "EasySpider.bat" to run our software instead.
+This version is for Windows 10 x64 and above.
 
+Please wait for at most 20 seconds if you see a white screen when open EasySpider.
 
-可以从其他机器导入任务，只需要把其他机器的tasks文件夹里的.json文件放入此目录的tasks文件夹里即可。同理执行号文件可以通过复制execution_instances文件夹中的.json文件来导入。注意，两个文件夹里的.json文件只支持命名为大于0的数字。
+Video Tutorial: https://youtube.com/playlist?list=PL0kEFEkWrT7mt9MUlEBV2DTo1QsaanUTp
+
+The software is totally not trojan/virus! If mistaken by antivirus software such as windows defender as a virus, please recover it, or open "EasySpider.bat" to run our software instead.
 
 Tasks can be imported from other machines by simply placing the .json files from the "tasks" folder of those machines into the "tasks" folder of this directory. Similarly, execution instance files can be imported by copying the .json files from the "execution_instances" folder. Note that only files named with a number greater than 0 are supported in both folders.

Releases/EasySpider_windows_amd64/tasks/13.json

@@ -0,0 +1 @@
+{"id":13,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"5/27/2023, 3:18:20 AM","version":"0.3.1","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://www.jd.com"}],"outputParameters":[{"id":0,"name":"自定义操作","desc":"自定义操作返回的数据","type":"string","exampleValue":""},{"id":1,"name":"自定义操作1","desc":"自定义操作返回的数据","type":"string","exampleValue":""}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2,3],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"codeMode":"1","code":"C:\\Users\\q9823\\Desktop/test.bat","waitTime":0,"recordASField":"1"}},{"id":3,"index":3,"parentId":0,"type":0,"option":5,"title":"自定义操作1","sequence":[],"isInLoop":false,"position":2,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":10,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"codeMode":0,"code":"return document.title=\"123\"","waitTime":0,"recordASField":"1"}}]}

Releases/EasySpider_windows_amd64/tasks/8.json

@@ -0,0 +1 @@
+{"id":8,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"5/27/2023, 2:49:57 AM","version":"0.3.1","containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://www.jd.com"}],"outputParameters":[{"id":0,"name":"参数1_页面标题","desc":"","type":"string","exampleValue":"京东全球版-专业的综合网上购物商城"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":30,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","wait":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":0,"contentType":7,"relative":false,"name":"参数1_页面标题","desc":"","extractType":0,"relativeXPath":"/html/body","allXPaths":["/html/body/div[4]","//div[contains(., '')]","id(\"header\")"],"exampleValues":[{"num":0,"value":"京东全球版-专业的综合网上购物商城"}],"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}]}}]}

Releases/EasySpider_windows_amd64/软件使用说明.txt

@@ -0,0 +1,14 @@
+欢迎将软件宣传给更多需要的朋友！
+
+官方网址: https://github.com/NaiboWang/EasySpider
+
+支持Windows 10 x64及以上版本。
+
+打开如果白屏请等待最多20秒，界面就会显示。
+
+视频教程：https://www.bilibili.com/video/BV1Fk4y1L7xX/
+
+这个软件绝对不是特洛伊木马/病毒！如果被像 Windows Defender 这样的杀毒软件误认为是病毒，请进行恢复，或者打开“EasySpider.bat”来运行我们的软件。
+
+可以从其他机器导入任务，只需要把其他机器的tasks文件夹里的.json文件放入此目录的tasks文件夹里即可。同理执行号文件可以通过复制execution_instances文件夹中的.json文件来导入。注意，两个文件夹里的.json文件只支持命名为大于0的数字。
+