Python全网页加载时间

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157

#!/usr/bin/env python3
#
#UserParameter=custom.page.load[*],/opt/sh/zbx_discover_site/page-load.py $1
#
import requests
from bs4 import BeautifulSoup
import re
import urllib.parse
import sys
from time import time

debug=1

class Loady:
    files = {
     'js' : {},
     'css' : {},
     'img' : {}
    }

    def __init__( self, url, headers = {} ):
        if not isinstance( headers, dict ):
            raise ValueError( 'Headers argument must be dict instance' )

        self.url = url
        self.total_time = 0
        self.js = []
        self.css = []
        self.img = []
        self.http_headers = headers
        self.soup = None
        self.total_size = 0

    def _get( self, tag ):
        """Gets all site additional files and prepares their URL to be loaded"""

        # Get current URL data
        domain_scheme, domain, _, _, _, _ = urllib.parse.urlparse( self.url )
        urls = []

        if tag == 'script':
            # Get all script tag with src attribute
            #			print(self.soup.find_all( 'script', { 'src' : re.compile( r'.*' ) } ))
            tags = self.soup.find_all( 'script', { 'src' : re.compile( r'.*' ) } )
        elif tag == 'img':
            #			print(self.soup.find_all( 'img', { 'src' : re.compile( r'.*' ) } ))
            tags = self.soup.find_all( 'img', { 'src' : re.compile( r'.*' ) } )
        # elif tag is 'i':
        # 	print(tags = self.soup.find_all('i', {'style': re.compile(r'.*')}))
        # 	tags = self.soup.find_all('i', {'style': re.compile(r'.*')})
        else:
            # Get all link tag with rel=stylesheet
            #			print(self.soup.find_all( 'link', { 'rel' : 'stylesheet' } ))
            tags = self.soup.find_all( 'link', { 'rel' : 'stylesheet' } )

        for each_tag in tags:
            # Get the value of src or href
            val = each_tag[ 'src' ] if tag == 'script' or tag == 'img' else each_tag[ 'href' ]
            #val = ''
            #if tag is 'script' or tag is 'img':
            #	val = each_tag['src']
            #else:
            #	val = each_tag['href']

            # parse the URL of the gotten URL
            url = urllib.parse.urlparse( val )

            if not url[ 0 ] and url[ 1 ]:
                # If URL has no scheme but has domain name, we assume it is a URL that supports HTTP(S). We just append the main site scheme to it
                if not val.startswith("//"):
                    urls.append( '{0}://{1}'.format( domain_scheme, val ) )
                else:
                    urls.append( '{0}:{1}'.format( domain_scheme, val ) )
            elif not url[ 1 ]:
                # URL has no domain, its a relative path. Append the domain name to it
                if not val.startswith("/"):
                    urls.append( '{0}://{1}/{2}'.format( domain_scheme, domain, val ) )
                else:
                    urls.append( '{0}://{1}{2}'.format( domain_scheme, domain, val ) )
            else:
                # Its an absolute path, no issues bro!
                urls.append( val )

        if tag == 'script':
            self.js = urls
        elif tag == 'img':
            self.img = urls
        else:
            self.css = urls

    def _load( self, t ):
        """Load the gotten links, check for response time and size. Appends it to self.files object"""
        _link_obj = []
        if t == 'script':
            _link_obj = self.js
        elif t == 'img':
            _link_obj = self.img
        else:
            _link_obj = self.css
#		for link in ( self.js if t is 'script' else self.css ):
        for link in (_link_obj):
            if debug == 1:
                print(link)
            try:
                start = time()
                r = requests.get( link )
                end = time()
                # Calculate the total time taken to load link
                response_time = ( end - start )
                # Page loaded successfully
                if r.status_code == 200:
                    # Get the size of page content
                    size = sys.getsizeof(r.content) if t == 'img' else sys.getsizeof(r.text)
                    # Add results to self.files object
                    obj = ''
                    if t == 'style':
                        obj = 'css'
                    elif t == 'img':
                        obj = 'img'
                    else:
                        obj = 'js'
                    self.files[obj][link] = {'byte_size': size, 'load_time': response_time}
                    # Sum up total time to the existing load time
                    self.total_time += response_time
                    self.total_size += size
            except Exception as e:
                if debug == 1:
                    print(e,link)
                continue

    def get( self ):
        """Loads the main website, calculate response time, page size and get additional files in site"""

        start = time()
        r = requests.get( self.url, headers = self.http_headers )
        stop = time()
        if r.status_code == 200:
            response = r.text
            self.total_time = self.total_time + ( stop - start )
            self.total_size += sys.getsizeof( response )
            self.soup = BeautifulSoup( response, 'html.parser' )

            self._get( 'script' )
            self._load( 'script' )
            self._get( 'style' )
            self._load( 'style' )
            self._get( 'img' )
            self._load('img')


load = Loady( sys.argv[1] , headers={ 'User-Agent' : 'zabbix pageload monitor' })
#load = Loady( sys.argv[1], headers={ 'User-Agent' : 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0' })
load.get()
#print("{TIME:\"",load.total_time,"\"}",sep='')
# print("%.3f"%load.total_time)
print( load.total_size ) # total load size
# print( load.files )   #load file and load size

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

#!/usr/bin/env python3
#discover site

file = open("/opt/sh/zbx_discover_site/site.txt")
print("{")
print("\t\"data\":[")
try:
	lines = file.readlines();
	count = 1
	for line in lines:
		line = line.strip("\n")
		print("\t\t{")
		print("\t\t\t\"{#SITE}\":\"",end='')
		print(line,end='')
		print("\"")
		print("\t\t}",end='')
		if count < len(lines):
			print(",")
		count = count + 1
finally:
	file.close()
print("\n\t]")
print("}")

1
2

https://www.example.com
http://www.example.com