Python爬取工控行業(yè)系統(tǒng)漏洞

nihao 發(fā)布于2019-07-31 10:30 / 1316人閱讀

摘要：先貼連接，讓各位觀眾老爺看看，對(duì)不對(duì)你們的胃口工控行業(yè)系統(tǒng)漏洞可以看到，這個(gè)網(wǎng)頁(yè)是靜態(tài)的，所以問(wèn)題變的非常的簡(jiǎn)單只需要用請(qǐng)求網(wǎng)頁(yè)就可以了話不多說(shuō)，直接貼代碼標(biāo)題公開(kāi)日期公開(kāi)日期危害級(jí)別危害級(jí)別影響產(chǎn)品影響產(chǎn)品漏洞描述漏洞

先貼連接，讓各位觀眾老爺看看，對(duì)不對(duì)你們的胃口

工控行業(yè)系統(tǒng)漏洞

可以看到，這個(gè)網(wǎng)頁(yè)是html靜態(tài)的，所以問(wèn)題變的非常的簡(jiǎn)單

只需要用request請(qǐng)求網(wǎng)頁(yè)就可以了

話不多說(shuō)，直接貼代碼

import requests
from urllib.parse import urlencode
from lxml import etree
import pymysql
import time
import xlwt
import xlrd


def makeurl():
    # http://ics.cnvd.org.cn/?tdsourcetag=s_pctim_aiomsg&max=20&offset=0
    baseurl = "http://ics.cnvd.org.cn/?"
    params = {
        "tdsourcetag": "s_pctim_aiomsg",
        "max": "20"
    }
    for page in range(MAX_PAGE):
        params["offset"] = page * 20
        url = baseurl + urlencode(params)
        print("url is ", url)
        yield url


def get_page_urllist(url):
    headers = {
        "Host": "ics.cnvd.org.cn",
        "Referer": "http://ics.cnvd.org.cn/?tdsourcetag=s_pctim_aiomsg&max=20&offset=40",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
    }
    response = requests.get(url, headers=headers)
    return response.text


def parse_urllist(content):
    html = etree.HTML(content)
    for li in html.xpath("http://tbody[@id="tr"]/tr"):
        yield li.xpath("td/a/@href")[0]


def get_page(url):
    headers = {
        "Host": "www.cnvd.org.cn",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
    }
    response = requests.get(url, headers=headers)
    return response.text


def parse_page(content, url):
    html = etree.HTML(content)
    item = {}
    item["url"] = url
    item["標(biāo)題"] = str(html.xpath("http://div[@class="blkContainerSblk"]/h1/text()")[0])

    item["CNVD_ID"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="CNVD-ID"]/following-sibling::*[1]//text()")])
    item["公開(kāi)日期"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="公開(kāi)日期"]/following-sibling::*[1]//text()")])
    item["危害級(jí)別"] = "".join([i.strip().replace(" ", "").replace("
", "").replace("
", "").replace("	", "") for i in
                            html.xpath("http://tbody/tr/td[text()="危害級(jí)別"]/following-sibling::*[1]//text()")])
    item["影響產(chǎn)品"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="影響產(chǎn)品"]/following-sibling::*[1]//text()")])
    try:
        item["BUGTRAQ_ID"] = "".join(
            [i.strip() for i in html.xpath("http://tbody/tr/td[text()="BUGTRAQ ID"]/following-sibling::*[1]//text()")])
    except:
        item["BUGTRAQ_ID"] = ""
    item["CVE_ID"] = "".join(
        [i.strip() for i in
         html.xpath("http://tbody/tr/td[text()="CVE ID"]/following-sibling::*[1]//text()")]) + " " + "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="CVE ID"]/following-sibling::*[1]//@href")])

    item["漏洞描述"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="漏洞描述"]/following-sibling::*[1]//text()")])
    item["漏洞類型"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="漏洞類型"]/following-sibling::*[1]//text()")])
    item["參考鏈接"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="參考鏈接"]/following-sibling::*[1]//text()")])
    item["漏洞解決方案"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="漏洞解決方案"]/following-sibling::*[1]//text()")])
    item["廠商補(bǔ)丁"] = "".join(
        [i.strip() for i in html.xpath(
            "http://tbody/tr/td[text()="廠商補(bǔ)丁"]/following-sibling::*[1]//text()")]) + " http://www.cnvd.org.cn" + "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="廠商補(bǔ)丁"]/following-sibling::*[1]//@href")])
    item["驗(yàn)證信息"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="驗(yàn)證信息"]/following-sibling::*[1]//text()")])
    item["報(bào)送時(shí)間"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="報(bào)送時(shí)間"]/following-sibling::*[1]//text()")])
    item["收錄時(shí)間"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="收錄時(shí)間"]/following-sibling::*[1]//text()")])
    item["更新時(shí)間"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="更新時(shí)間"]/following-sibling::*[1]//text()")])
    item["漏洞附件"] = "".join(
        [i.strip() for i in html.xpath("http://tbody/tr/td[text()="漏洞附件"]/following-sibling::*[1]//text()")])

    return item


def save_data(index, item, workbook):
    sheet = workbook.get_sheet("sheet1")  # 創(chuàng)建一個(gè)sheet表格
    for col, value in enumerate(item.values()):
        sheet.write(index, col, value)
    workbook.save(filename)
    print("保存成功")


def excel_prepare(heads):
    workbook = xlwt.Workbook()
    sheet = workbook.add_sheet("sheet1", cell_overwrite_ok=True)  # 創(chuàng)建一個(gè)sheet表格
    for col, value in enumerate(heads):
        sheet.write(0, col, value)
    return workbook


def urlisexist(url, urlset):
    if url in urlset:
        return True
    else:
        return False


def getallurl(filename):
    workbook = xlrd.open_workbook(filename)
    sheet1 = workbook.sheet_by_name("sheet1")
    results = sheet1.col_values(0, 1)
    return results


def read_old(filename):
    workbook = xlrd.open_workbook(filename)
    sheet1 = workbook.sheet_by_name("sheet1")
    alloldset = []
    for index in range(sheet1.nrows):
        alloldset.append(sheet1.row_values(index))
    return alloldset, sheet1.nrows


def save_old(index, olditem):
    sheet = workbook.get_sheet("sheet1")  # 創(chuàng)建一個(gè)sheet表格
    for col, value in enumerate(olditem):
        sheet.write(index, col, value)
    workbook.save(filename)


if __name__ == "__main__":
    # http://ics.cnvd.org.cn/?tdsourcetag=s_pctim_aiomsg&max=20&offset=0

    # 睡眠時(shí)間
    TIMESLEEP = 0

    filename = "工程控制系統(tǒng)漏洞.xls"

    MAX_PAGE = 96



    heads = ["url",
             "標(biāo)題",
             "CNVD_ID",
             "公開(kāi)日期",
             "危害級(jí)別",
             "影響產(chǎn)品",
             "BUGTRAQ_ID",
             "CVE_ID",
             "漏洞描述",
             "漏洞類型",
             "參考鏈接",
             "漏洞解決方案",
             "廠商補(bǔ)丁",
             "驗(yàn)證信息",
             "報(bào)送時(shí)間",
             "收錄時(shí)間",
             "更新時(shí)間",
             "漏洞附件"]

    try:
        alloldset, length = read_old(filename)
    except:
        alloldset = []
        length = 1

    workbook = excel_prepare(heads)

    for index, olditem in enumerate(alloldset):
        save_old(index, olditem)

    try:
        urlset = getallurl(filename)
    except:
        urlset = []

    index = length
    for urlofpage in makeurl():
        pagelistcontent = get_page_urllist(urlofpage)
        for url in parse_urllist(pagelistcontent):
            print("url is >>>", url)
            if not urlisexist(url, urlset):
                time.sleep(TIMESLEEP)
                result = get_page(url)
                item = parse_page(result, url)
                print("item is >>>", item)
                save_data(index, item, workbook)
                index = index + 1

    workbook.save(filename)

不懂的地方，下方評(píng)論提問(wèn)

云服務(wù)器 GPU云服務(wù)器工控網(wǎng)關(guān)python linux系統(tǒng)工控機(jī) python爬取 python爬取數(shù)據(jù)

文章版權(quán)歸作者所有，未經(jīng)允許請(qǐng)勿轉(zhuǎn)載,若此文章存在違規(guī)行為，您可以聯(lián)系管理員刪除。

轉(zhuǎn)載請(qǐng)注明本文地址：http://www.ezyhdfw.cn/yun/43933.html

發(fā)表評(píng)論

登陸后可評(píng)論

0條評(píng)論

nihao

男|高級(jí)講師

我要關(guān)注我要私信

TA的文章

LightNode：感恩&黑五雙節(jié)狂歡,送9.9美金,新加坡vps/香港vps,香港原生IP/CN2

閱讀 3122·2021-11-24 10:22
TMThosting：黑五特賣會(huì),美國(guó)西雅圖vps低至月付55折,全場(chǎng)獨(dú)立服務(wù)器9折

閱讀 3126·2021-11-23 10:10
【沁恒CH32學(xué)習(xí)】——RISC-V架構(gòu)學(xué)習(xí)筆記

閱讀 1459·2021-09-28 09:35
面試--css實(shí)現(xiàn)元素的水平和垂直居中

閱讀 1818·2019-08-29 13:16
vue源碼分析系列之入debug環(huán)境搭建

閱讀 1467·2019-08-26 13:29
MongoDB學(xué)習(xí)筆記（1）- MongoDB簡(jiǎn)介、數(shù)據(jù)類型及幫助命令

閱讀 2862·2019-08-26 10:27
面對(duì)對(duì)象（二）

閱讀 739·2019-08-26 10:09
spy-debugger + Charles 移動(dòng)端調(diào)試

閱讀 1526·2019-08-23 18:05

亚洲中字慕日产2020,大陆极品少妇内射AAAAAA,无码av大香线蕉伊人久久,久久精品国产亚洲av麻豆网站

資訊專欄INFORMATION COLUMN

上云采購(gòu)季！| 2核2G4M爆款云服務(wù)器低至59元/年，更有多臺(tái)、長(zhǎng)期優(yōu)惠，快來(lái)選購(gòu)！

Python爬取工控行業(yè)系統(tǒng)漏洞

相關(guān)文章

**立根鑄魂，麒麟信安攜手歐拉共推操作系統(tǒng)產(chǎn)業(yè)新發(fā)展**

**#yyds干貨盤點(diǎn)#安全漏洞頻出，信息系統(tǒng)中安全防護(hù)如何有效構(gòu)建？**

發(fā)表評(píng)論

0條評(píng)論

nihao

男|高級(jí)講師

TA的文章

LightNode：感恩&黑五雙節(jié)狂歡,送9.9美金,新加坡vps/香港vps,香港原生IP/CN2

TMThosting：黑五特賣會(huì),美國(guó)西雅圖vps低至月付55折,全場(chǎng)獨(dú)立服務(wù)器9折

【沁恒CH32學(xué)習(xí)】——RISC-V架構(gòu)學(xué)習(xí)筆記

面試--css實(shí)現(xiàn)元素的水平和垂直居中

vue源碼分析系列之入debug環(huán)境搭建

MongoDB學(xué)習(xí)筆記（1）- MongoDB簡(jiǎn)介、數(shù)據(jù)類型及幫助命令

面對(duì)對(duì)象（二）

spy-debugger + Charles 移動(dòng)端調(diào)試

最新活動(dòng)

資訊專欄INFORMATION COLUMN

上云采購(gòu)季！| 2核2G4M爆款云服務(wù)器低至59元/年，更有多臺(tái)、長(zhǎng)期優(yōu)惠，快來(lái)選購(gòu)！

Python爬取 工控行業(yè)系統(tǒng)漏洞

相關(guān)文章

發(fā)表評(píng)論

0條評(píng)論

男|高級(jí)講師

TA的文章

最新活動(dòng)

上云采購(gòu)季！| 2核2G4M爆款云服務(wù)器低至59元/年，更有多臺(tái)、長(zhǎng)期優(yōu)惠，快來(lái)選購(gòu)！

Python爬取工控行業(yè)系統(tǒng)漏洞