URL pattern detect

ElectronJS/config.json

@@ -1,7 +1 @@
-{
-  "webserver_address": "http://localhost",
-  "webserver_port": 8074,
-  "user_data_folder": "./user_data",
-  "debug": true,
-  "absolute_user_data_folder": "D:\\Document\\Projects\\EasySpider\\ElectronJS\\user_data"
-}
+{"webserver_address":"http://localhost","webserver_port":8074,"user_data_folder":"./user_data","debug":true,"absolute_user_data_folder":"D:\\Document\\Projects\\EasySpider\\ElectronJS\\user_data"}

ElectronJS/main.js

@@ -457,7 +457,7 @@ function handleOpenBrowser(event, lang = "en", user_data_folder = "", mobile = f
     runBrowser(lang, user_data_folder, mobile);
     let size = screen.getPrimaryDisplay().workAreaSize;
     let width = parseInt(size.width);
-    let height = parseInt(size.height * 0.65);
+    let height = parseInt(size.height * 0.6);
     flowchart_window = new BrowserWindow({
         x: 0,
         y: 0,

ElectronJS/src/taskGrid/FlowChart_CN.html

@@ -513,25 +513,33 @@
                     <h4 class="modal-title" id="myModalLabel">保存任务</h4>
                     <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
                 </div>
-                <div class="modal-body">
+                <div class="modal-body" style="height:400px;overflow: auto">
                     <input onkeydown="inputDelete(event)" id="serviceId" type="hidden" name="serviceId" value="-1"></input>
                     <input onkeydown="inputDelete(event)" id="url" type="hidden" name="url" value="about:blank"></input>
                     <label>任务名称：</label>
                     <input onkeydown="inputDelete(event)" required name="serviceName" value="新web采集任务" id="serviceName" class="form-control"></input>
                     <label>任务描述：</label>
                     <input onkeydown="inputDelete(event)" id="serviceDescription" name="serviceDescription" class="form-control"></input>
-                    <label>每采集多少条数据保存一次（值越大采集速度越快，但如果意外退出则有数据丢失风险）：</label>
-                    <input onkeydown="inputDelete(event)" type="number" value="10" id="saveThreshold" name="saveThreshold" class="form-control"></input>
-                    <label>是否为cloudflare等极端反爬网站：</label>
-                    <select id="cloudflare" name="cloudflare" class="form-control">
-                        <option value = 0>否</option>
-                        <option value = 1>是</option>
+                    <label>导出数据格式：</label>
+                    <select id="outputFormat" class="form-control">
+                        <option value = "csv">CSV</option>
+                        <option value = "xlsx">XLSX（EXCEL）</option>
+                        <option value = "mysql">MySQL</option>
                     </select>
                     <label>浏览器模拟类型：</label>
                     <select id="environment" name="environment" class="form-control">
                         <option value = 0>电脑端</option>
                         <option value = 1>手机端（Cloudflare模式下不支持）</option>
                     </select>
+                    <label>是否为cloudflare等极端反爬网站：</label>
+                    <select id="cloudflare" name="cloudflare" class="form-control">
+                        <option value = 0>否</option>
+                        <option value = 1>是</option>
+                    </select>
+                    <label>每采集多少条数据保存一次（值越大采集速度越快，但如果意外退出则有数据丢失风险）：</label>
+                    <input onkeydown="inputDelete(event)" type="number" value="10" id="saveThreshold" name="saveThreshold" class="form-control"></input>
+                    <label>控制台预览时数据最大显示长度：</label>
+                    <input onkeydown="inputDelete(event)" type="number" value="15" id="maxViewLength" class="form-control"></input>
 
                 </div>
                 <div class="modal-footer">

ElectronJS/src/taskGrid/logic_CN.js

@@ -427,6 +427,8 @@ function saveService(type) {
             "saveThreshold": saveThreshold,
             "cloudflare": cloudflare,
             "environment": environment,
+            "maxViewLength": parseInt($("#maxViewLength").val()),
+            "outputFormat": $("#outputFormat").val(),
             "containJudge": containJudge,
             "desc": serviceDescription,
             "inputParameters": inputParameters,
@@ -460,10 +462,24 @@ if (sId != null && sId != -1) //加载任务
     $.get(backEndAddressServiceWrapper + "/queryTask?id=" + sId, function(result) {
         nodeList = result["graph"];
         app.$data.list.nl = nodeList;
+        for(let node of nodeList){ //兼容旧版本
+            if(node["option"] == 1){
+                if(!("cookies" in node["parameters"])) {
+                    node["parameters"]["cookies"] = "";
+                }
+            }
+        }
         $("#serviceName").val(result["name"]);
         $("#serviceId").val(result["id"]);
         $("#url").val(result["url"]);
         $("#serviceDescription").val(result["desc"]);
+        for(let key of Object.keys(result)){
+            try{
+                $("#"+key).val(result[key]);
+            } catch(e){
+                console.log(e);
+            }
+        }
         refresh();
     });
 } else {

ElectronJS/tasks/143.json

@@ -0,0 +1 @@
+{"id":143,"name":"中国知网","url":"https://chn.oversea.cnki.net/index/","links":"https://chn.oversea.cnki.net/index/","create_time":"7/6/2023, 4:50:52 AM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"outputFormat":"csv","containJudge":false,"desc":"https://chn.oversea.cnki.net/index/","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://chn.oversea.cnki.net/index/","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://chn.oversea.cnki.net/index/"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://chn.oversea.cnki.net/index/","links":"https://chn.oversea.cnki.net/index/","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}}]}

ExecuteStage/.vscode/launch.json

@@ -10,9 +10,9 @@
             "program": "${file}",
             "console": "integratedTerminal",
             "justMyCode": true,
-             "args": ["--id", "[3]", "--read_type", "remote", "--headless", "0"]
-            // "args": ["--id", "[2]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"]
-            // "args": ["--id", "[44]", "--headless", "0", "--user_data", "1"]
+            //  "args": ["--id", "[7]", "--read_type", "remote", "--headless", "0"]
+            // "args": ["--id", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"]
+            "args": ["--id", "[16]", "--headless", "0", "--user_data", "1"]
         }
     ]
 }

ExecuteStage/config.json

@@ -1,6 +1,6 @@
 {
-    "webserver_address": "http://localhost",
-    "webserver_port": 8074,
-    "user_data_folder": "./user_data",
-    "absolute_user_data_folder": "D:\\Documents\\Projects\\EasySpider\\.temp_to_pub\\EasySpider_windows_x64\\user_data"
-}
+  "webserver_address": "http://localhost",
+  "webserver_port": 8074,
+  "user_data_folder": "./user_data",
+  "absolute_user_data_folder": "D:\\Documents\\Projects\\EasySpider\\ElectronJS\\user_data"
+}

ExecuteStage/easyspider_executestage.py

@@ -31,6 +31,8 @@ from selenium.webdriver import ActionChains
 from selenium.webdriver.common.by import By
 import undetected_chromedriver as uc
 import random
+# import pandas as pd
+from openpyxl import load_workbook, Workbook
 # import numpy
 import csv
 import os
@@ -40,15 +42,16 @@ from PIL import Image
 # import uuid
 from threading import Thread, Event
 from myChrome import MyChrome
-from utils import check_pause, download_image, get_output_code, isnull
+from utils import check_pause, download_image, get_output_code, isnull, write_to_csv, write_to_excel
 desired_capabilities = DesiredCapabilities.CHROME
 desired_capabilities["pageLoadStrategy"] = "none"
 
 
 class BrowserThread(Thread):
-    def __init__(self, browser_t, id, service, version, event):
+    def __init__(self, browser_t, id, service, version, event, config):
         Thread.__init__(self)
         self.browser = browser_t
+        self.config = config
         self.id = id
         self.event = event
         self.saveName = saveName
@@ -65,6 +68,14 @@ class BrowserThread(Thread):
         WebDriverWait(self.browser, 10)
         self.browser.get('about:blank')
         self.procedure = service["graph"]  # 程序执行流程
+        try:
+            self.maxViewLength = service["maxViewLength"]  # 最大显示长度
+        except:
+            self.maxViewLength = 15
+        try:
+            self.outputFormat = service["outputFormat"]  # 输出格式
+        except:
+            self.outputFormat = "csv"
         try:
             if service["version"] >= "0.3.1":  # 0.3.1及以上版本以上的EasySpider兼容从0.3.1版本开始的所有版本
                 pass
@@ -88,6 +99,7 @@ class BrowserThread(Thread):
         self.links = list(
             filter(isnull, service["links"].split("\n")))  # 要执行的link的列表
         self.OUTPUT = []  # 采集的数据
+        self.OUTPUT.append([])  # 添加表头
         self.containJudge = service["containJudge"]  # 是否含有判断语句
         tOut = service["outputParameters"]  # 生成输出参数对象
         self.outputParameters = {}
@@ -95,15 +107,19 @@ class BrowserThread(Thread):
         self.log = ""  # 记下现在总共开了多少个标签页
         self.history = {"index": 0, "handle": None}  # 记录页面现在所以在的历史记录的位置
         self.SAVED = False  # 记录是否已经存储了
-        # 文件叠加的时候不添加表头
-        if not os.path.exists("Data/Task_" + str(self.id) + "/" + self.saveName + '.csv'):
-            self.OUTPUT.append([])  # 添加表头
         for para in tOut:
             if para["name"] not in self.outputParameters.keys():
                 self.outputParameters[para["name"]] = ""
                 self.dataNotFoundKeys[para["name"]] = False
-                if not os.path.exists("Data/Task_" + str(self.id) + "/" + self.saveName + '.csv'):
-                    self.OUTPUT[0].append(para["name"])
+                # 文件叠加的时候不添加表头
+                if self.outputFormat == "csv":
+                    if not os.path.exists("Data/Task_" + str(self.id) + "/" + self.saveName + '.csv'):
+                        self.OUTPUT[0].append(para["name"])
+                elif self.outputFormat == "xlsx":
+                    if not os.path.exists("Data/Task_" + str(self.id) + "/" + self.saveName + '.xlsx'):
+                        self.OUTPUT[0].append(para["name"])
+                elif self.outputFormat == "mysql":  # MySQL不需要表头
+                    pass
         self.urlId = 0  # 全局记录变量
         self.preprocess()  # 预处理，优化提取数据流程
 
@@ -134,6 +150,8 @@ class BrowserThread(Thread):
     def run(self):
         # 挨个执行程序
         for i in range(len(self.links)):
+            print("正在执行第", i + 1, "/ ", len(self.links), "个链接")
+            print("Executing link", i + 1, "/ ", len(self.links))
             self.executeNode(0)
             self.urlId = self.urlId + 1
         files = os.listdir("Data/Task_" + str(self.id) + "/" + self.saveName)
@@ -167,11 +185,17 @@ class BrowserThread(Thread):
             with open("Data/Task_" + str(self.id) + "/" + self.saveName + '_log.txt', 'a', encoding='utf-8-sig') as file_obj:
                 file_obj.write(self.log)
                 file_obj.close()
-            with open("Data/Task_" + str(self.id) + "/" + self.saveName + '.csv', 'a', encoding='utf-8-sig', newline="") as f:
-                f_csv = csv.writer(f)
-                for line in self.OUTPUT:
-                    f_csv.writerow(line)
-                f.close()
+            if self.outputFormat == "csv":
+                file_name = "Data/Task_" + \
+                    str(self.id) + "/" + self.saveName + '.csv'
+                write_to_csv(file_name, self.OUTPUT)
+            elif self.outputFormat == "xlsx":
+                file_name = "Data/Task_" + \
+                    str(self.id) + "/" + self.saveName + '.xlsx'
+                write_to_excel(file_name, self.OUTPUT)
+            elif self.outputFormat == "mysql":
+                # write_to_mysql(self.config, )
+                pass
             self.OUTPUT = []
             self.log = ""
 
@@ -302,7 +326,7 @@ class BrowserThread(Thread):
             line = []
             for value in self.outputParameters.values():
                 line.append(value)
-                print(value[:15], " ", end="")
+                print(value[:self.maxViewLength], " ", end="")
             print("")
             self.OUTPUT.append(line)
 
@@ -728,6 +752,9 @@ class BrowserThread(Thread):
                 self.browser.execute_script('window.stop()')
             except:
                 pass
+        except Exception as e:
+            print("Failed to load page: " + url)
+            self.recordLog('Failed to load page: ' + url)
         try:
             self.history["index"] = self.browser.execute_script(
                 "return history.length")
@@ -1184,7 +1211,7 @@ class BrowserThread(Thread):
         line = []
         for value in self.outputParameters.values():
             line.append(value)
-            print(value[:15], " ", end="")
+            print(value[:self.maxViewLength], " ", end="")
         print("")
         self.OUTPUT.append(line)
         # rt.end()
@@ -1279,12 +1306,15 @@ if __name__ == '__main__':
     # 2. User Profile文件夹的路径是：C:\Users\用户名\AppData\Local\Google\Chrome\User Data不要加Default
     # 3. 就算User Profile相同，chrome版本不同所存储的cookie信息也不同，也不能爬
     # 4. TMALL如果一直弹出验证码，而且无法通过验证，那么需要在其他浏览器上用
-    if c.user_data:
+    try:
         with open(c.config_folder + c.config_file_name, "r", encoding='utf-8') as f:
             config = json.load(f)
             absolute_user_data_folder = config["absolute_user_data_folder"]
             print("\nAbsolute_user_data_folder:",
                   absolute_user_data_folder, "\n")
+    except:
+        pass
+    if c.user_data:
         option.add_argument(
             f'--user-data-dir={absolute_user_data_folder}')  # TMALL 反扒
         option.add_argument("--profile-directory=Default")
@@ -1371,7 +1401,8 @@ if __name__ == '__main__':
             print("过Cloudflare验证模式")
         event = Event()
         event.set()
-        thread = BrowserThread(browser_t, i, service, c.version, event)
+        thread = BrowserThread(browser_t, i, service,
+                               c.version, event, config=config)
         print("Thread with task id: ", i, " is created")
         threads.append(thread)
         thread.start()

ExecuteStage/requirements.txt

@@ -5,4 +5,5 @@ pyinstaller
 Pillow
 pytesseract
 keyboard
-undetected_chromedriver
+undetected_chromedriver
+openpyxl

ExecuteStage/utils.py

@@ -1,10 +1,21 @@
 # 控制流程的暂停和继续
 
+import csv
 import os
 import time
 import uuid
 import keyboard
+from openpyxl import Workbook, load_workbook
 import requests
+from urllib.parse import urlparse
+
+
+def is_valid_url(url):
+    try:
+        result = urlparse(url)
+        return all([result.scheme, result.netloc])
+    except ValueError:
+        return False
 
 
 def check_pause(key, event):
@@ -28,27 +39,32 @@ def download_image(url, save_directory):
     headers = {
         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
     }
+    if is_valid_url(url):
+        # 发送 GET 请求获取图片数据
+        response = requests.get(url, headers=headers)
 
-    # 发送 GET 请求获取图片数据
-    response = requests.get(url, headers=headers)
+        # 检查响应状态码是否为成功状态
+        if response.status_code == requests.codes.ok:
+            # 提取文件名
+            file_name = url.split('/')[-1].split("?")[0]
 
-    # 检查响应状态码是否为成功状态
-    if response.status_code == requests.codes.ok:
-        # 提取文件名
-        file_name = url.split('/')[-1].split("?")[0]
+            # 生成唯一的新文件名
+            new_file_name = file_name + '_' + \
+                str(uuid.uuid4()) + '_' + file_name
 
-        # 生成唯一的新文件名
-        new_file_name = file_name + '_' + str(uuid.uuid4()) + '_' + file_name
+            # 构建保存路径
+            save_path = os.path.join(save_directory, new_file_name)
 
-        # 构建保存路径
-        save_path = os.path.join(save_directory, new_file_name)
+            # 保存图片到本地
+            with open(save_path, 'wb') as file:
+                file.write(response.content)
 
-        # 保存图片到本地
-        with open(save_path, 'wb') as file:
-            file.write(response.content)
-
-        print("图片已成功下载到:", save_path)
-        print("The image has been successfully downloaded to:", save_path)
+            print("图片已成功下载到:", save_path)
+            print("The image has been successfully downloaded to:", save_path)
+        else:
+            print("下载图片失败，请检查此图片链接是否有效:", url)
+            print(
+                "Failed to download image, please check if this image link is valid:", url)
     else:
         print("下载图片失败，请检查此图片链接是否有效:", url)
         print("Failed to download image, please check if this image link is valid:", url)
@@ -71,6 +87,29 @@ def isnull(s):
     return len(s) != 0
 
 
+def write_to_csv(file_name, data):
+    with open(file_name, 'a', encoding='utf-8-sig', newline="") as f:
+        f_csv = csv.writer(f)
+        for line in data:
+            f_csv.writerow(line)
+        f.close()
+
+
+def write_to_excel(file_name, data):
+    if os.path.exists(file_name):
+        # 加载现有的工作簿
+        wb = load_workbook(file_name)
+        ws = wb.active
+    else:
+        # 创建新的工作簿和工作表
+        wb = Workbook()
+        ws = wb.active
+    # 追加数据到工作表
+    for line in data:
+        ws.append(line)
+    # 保存工作簿
+    wb.save(file_name)
+
 
 class Time:
     def __init__(self, type1=""):