一个 activemq 的爬虫脚本

实现：当“number of pendig messages”的数值超过 100 时，发送钉钉群通知

# encoding=utf-8
import urllib2
import cookielib
from bs4 import BeautifulSoup
import re
import requests
# import urllib
import json
queue_url1 = "http://127.0.0.1:8161/admin/queues.jsp"
queue_url2 = "http://127.0.0.1:8162/admin/queues.jsp"
def MQBrowser(url):
#    login_page = queue_url
    login_page = url
    try:
        cj = cookielib.CookieJar()
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))

        opener.addheaders = [('User-agent', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)'),
                             ('Authorization', 'Basic eG1sbHltcTpYTWxpYW5sdW95aW1xISE=')]
        #ata = urllib.urlencode({"user":'xmllymq',"password":'XMlianluoyimq!!'})  
        opener.open(url)
        op = opener.open(login_page)
        data = op.read()
        return data
    except Exception,e:
        print(str(e))


#def get_queue_size_by_name(queue_name):
#    content = MQBrowser(queue_url)
#    soup = BeautifulSoup(''.join(content),'lxml')
#    queue_list = soup.findAll('tbody')[1].findAll('tr')
#    for i in queue_list:
#        queue_info = i.findAll('td')
#        if queue_name == queue_info[0].text.strip():
#            num_pending = queue_info[1].text.strip()
#    return -1


ef dingding(Queue_Name,Num_Pending):

    url = 'https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxx'
    headers = {
            "Content-Type": "application/json",
            "Chartset": "utf-8"
    }

    # 要发送的文本是json格式
    request_data = {
            "msgtype": "markdown",
             "markdown": {"title": "告警：队列消息积压",
                          "text": "消息队列可能消费异常：\n\n"+"Queue_Name:%s \n\n" %Queue_Name +"Num_Pending:%s \n\n" %Num_Pending
              },
             "at": {
                    "atMobiles": [],
                    "isAtAll": False
              }
    }

    # 把json转变为字符串格式数据
    send_data = json.dumps(request_data)
    # 这个是发送post请求，请求钉钉接口
    response = requests.post(url=url, headers=headers, data=send_data)
    # 讲求成功后返回的数据
    content = response.content.decode()
    # 打印
    # print(content)
  
 def get_queue_size(queue_url):
    content = MQBrowser(queue_url)
    soup = BeautifulSoup(''.join(content),'lxml')
    #li = soup.find('li')
    queue_list = soup.findAll('tbody')[1].findAll('tr')
    queue_dic = {}
    for i in queue_list:
        queue_info = i.findAll('td')
        queue_name = queue_info[0].text.strip()
        num_pending = queue_info[1].text.strip()
        num_consumer = queue_info[2].text.strip()
        num_enqueued = queue_info[3].text.strip()
        num_dequeued = queue_info[4].text.strip()
        queue_dic[queue_name] = [num_pending, num_consumer, num_enqueued, num_dequeued]
        if (int(num_pending.encode("utf-8")) > 100):
           dingding(queue_name,num_pending)
    #print(queue_dic[1][1])


#def get_queue_detail(queue_name):
#    content = MQBrowser("http://127.0.0.1:8161/admin/browse.jsp?JMSDestination=%s" % queue_name)
#    soup = BeautifulSoup(''.join(content),'lxml')


if __name__ == "__main__":
    import sys
    reload(sys)
    sys.setdefaultencoding('utf8')
    #MQBrowser(queue_url)
    get_queue_size(queue_url1)
    get_queue_size(queue_url2)
​