admin/OSSAnalyser
1
0
Fork 0
mirror of https://github.com/HusseinBakri/OSSAnalyser.git synced 2026-08-14 00:58:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add files via upload

Browse Source
This commit is contained in:
Hussein Bakri
2016-09-20 12:04:01 +01:00
committed by GitHub
parent c42392e2ba
commit e8aef3a5bd
3 changed files with 266 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ParseHTMLToCSVsimstatsajax.py
+120
View File
@@ -0,0 +1,120 @@
#!/bin/python3
#################################################################################################################################################
#Author: Hussein Bakri
#Script Title: Parses HTML files from Web Statistics AJAX module - simstatsajax- into ONE CSV file for later statistical analysis.
#License: GNU GPL v3 License - you are free to distribute, change, enhance and include any of the code of this script in your tools. I only expect #adequate attribution of this work. The attribution should include the title of the script, the author and the site or the document where the #script is taken from.
#Python 3 is needed
#This script Parses the HTML files obtained and stored in SStats_simstatsajax and transform all of them into one CSV file for statistical analysis
#It utilizes the BeautifulSoup Python module (bs4) which needs to be installed (on Linux: sudo pip3 install BeautifulSoup)
#It utilizes also the csv module which needs to be available
#-----IN EACH HTML - the 'td' HTML tag is what important to get---------------
#By assigning the following:
#table = SoupObj.find('table')
#rows = table.findAll('tr')
#cols = table.findAll('td')
#from colum 0 till 9 correspond from Dilatn to ScrLPS
#colum 10 till 19 correspond to values under them
#from colum 20 till 28 correspond from Dilatn to ScrLPS
#colum 29 till 37 correspond to values under them
#
#################################################################################################################################################
import bs4, csv, time
import itertools as it
print('Parsing the HTMLs and storing them into one CSV: output.csv...')
print()
#time.sleep(2)
#creating a CSV file and writing a header row
print('Creating the CSV file named: output.csv')
outputfile = open('output.csv', 'w', newline='')
outputWriter = csv.writer(outputfile)
print()
print('\n......................................................')
print()
print('Processing first HTML file which should be 0.html, please wait...')
loadedHTMLFile = open('0.html')
SoupObj = bs4.BeautifulSoup(loadedHTMLFile.read())
Alltables = SoupObj.findAll('table')
rowsOfFirstTable = Alltables[0].findAll('tr')
colsOfFirstTable = Alltables[0].findAll('td')
RegionNames = SoupObj.findAll('h2')
print()
print()
#Writing the header into header list
header=[]
print(range(0,len(RegionNames)))
for j in range(0,len(RegionNames)):
for i in it.chain(range(0, 10), range(20, 29)):
#Writing the header columns names
print(colsOfFirstTable[i].string) # Write this to CSV
header.append(RegionNames[j].getText() + ' ' + colsOfFirstTable[i].string)
#Writing the header file of the CSV
print('Writing the header of the CSV file, please wait....')
outputWriter.writerow(header)
print('Fetching the values of this header, please wait...')
valuesRow1=[]
for j in range(0,len(RegionNames)):
for i in it.chain(range(10, 20), range(29, 38)):
#Writing the values of this HTML under exactly each colum
CurrentCols = Alltables[j].findAll('td')
print(CurrentCols[i].string) # Write this to CSV
valuesRow1.append(CurrentCols[i].string)
#Writing the values to the CSV
print('Writing the values to the CSV file, please wait....')
outputWriter.writerow(valuesRow1)
print('Storing other rows from other HTML files into the CSV. Please wait....')
for files in range(1,180): # 3+ time of inner loop in seconds
loadedHTMLFile = open('%s.html' % (files))
print('Storing content of:' + '%s.html' % (files))
time.sleep(1)
SoupObj = bs4.BeautifulSoup(loadedHTMLFile.read())
valuesRow=[]
Alltables = SoupObj.findAll('table')
#colsOfFirstTable = Alltables[0].findAll('td')
time.sleep(1)
for j in range(0,len(RegionNames)):
for i in it.chain(range(10, 20), range(29, 38)):
#Writing the values of this HTML under exactly each colum
CurrentCols = Alltables[j].findAll('td')
print(CurrentCols[i].string) # Write this to CSV
valuesRow.append(CurrentCols[i].string)
print(valuesRow) # Write this to CSV
outputWriter.writerow(valuesRow)
time.sleep(1)
print('Done')
loadedHTMLFile.close()
outputfile.close()
SStatscript
+61
View File
@@ -0,0 +1,61 @@
#!/bin/bash
##################################################################################################################################################
#Author: Hussein Bakri
#Script Title: Method of getting stats from the OpenSim AJAX Web Statistical module and storing them in HTML files
#License: GNU GPL v3 License - you are free to distribute, change, enhance and include any of the code of this script in your tools. I only expect #adequate attribution of this work. The attribution should include the title of the script, the author and the site or the document where the #script is taken from.
#The Web Statistics Module provides region statistics information. The data is provided as AJAX html page which automatically updates over time. #The page is provided by the internal OpenSimulator web server. The module also stores historical data which is displayed on these web pages.
#----------- Retrieve HTML files from Web Statistics AJAX module -simstatsajax/activelogajax/activeconnectionsajax---------
#This Bash shell script saves statistics every 1 second for 3 minutes (during 180 seconds) from the OpenSim Web Statistics Module
#OpenSim Wiki article : http://opensimulator.org/wiki/Web_Statistics_Module
#The challenge solved is to get the XHR calls URLs of the AJAX page - This was done through using the Developer tools of Web browsers mainly the
#Network Inspectors. The 3 XHR call URLS are being fetched regularly on the AJAX page:
#http://0.0.0.0:9000/SStats/simstatsajax.html - have useful Simulator stats
#http://0.0.0.0:9000/SStats/activelogajax.html - NOTHING USEFUL HERE. It is retrieved for completion sake
#http://0.0.0.0:9000/SStats/activeconnectionsajax.html - has useful stats about packets in the different channels of circuits of each region.
#The Shell script fetches XHR URLs using wget command and stores the output as HTML files in seperate folders (3 folders)
#Python script(s) will later -in each folder- parse the HTML files and store their content into ONE CSV file for later statistical Analysis.
#This script will generate many files and take some time....
##################################################################################################################################################
echo "Let us begin the retrieval of statistics Through Web Statistics Module in the OpenSim server..."
echo
echo "Retrieval is done every 1 second for 3 minutes or during 180 seconds (i.e 180 values intake)"
echo
echo " It is done through the retrival of HTML files from the HTTP Server (Web Statistics Module) of OpenSim through the wget command..."
echo " on http://<OPENSIMSERVERIP>:9000/SStats/"
echo
echo
echo
echo "Please go and launch any avatar mobility script you want - you can not do anything here if you want, just press any key ..."
sleep 2
read -rsp $'When ready, Press any key in this terminal to continue - terminal should be in focus...\n' -n1 key
echo "Creating a folder named SStats_simstatsajax (if it does not exist)...."
mkdir -p SStats_simstatsajax
echo
echo "Creating a folder named SStats_activelogajax (if it does not exist)...."
mkdir -p SStats_activelogajax
echo
echo "Creating a folder named SStats_activeconnectionsajax (if it does not exist)...."
mkdir -p SStats_activeconnectionsajax
echo
filenumber=0
while [ $filenumber -lt 180 ]
do
wget http://0.0.0.0:9000/SStats/simstatsajax.html -O SStats_simstatsajax/${filenumber}.html
wget http://0.0.0.0:9000/SStats/activelogajax.html -O SStats_activelogajax/${filenumber}.html
wget http://0.0.0.0:9000/SStats/activeconnectionsajax.html -O SStats_activeconnectionsajax/${filenumber}.html
sleep 1
((filenumber++))
done
echo "Retrieval Finished ...."
echo
echo "..."
echo "Bye"
requestsTest.py
+85
View File
@@ -0,0 +1,85 @@
#!/bin/python3
#################################################################################################################################################
#Author: Hussein Bakri
#Script Title: Retrieve HTML files from Web Statistics AJAX module - simstatsajax
#License: GNU GPL v3 License - you are free to distribute, change, enhance and include any of the code of this script in your tools. I only expect #adequate attribution of this work. The attribution should include the title of the script, the author and the site or the document where the #script is taken from.
#----------
#
#Python 3 is needed
#This script transform Transform JSON files that Web Statistics AJAX module through UXSimStatus into ONE CSV file for later statistical analysis
# It uses the Request module to request every 1 second for 3 minutes (180 seconds) http://0.0.0.0:9000/SStats/simstatsajax.html'
# It Implements another shell script named: "SStatscript" using the Python requests module
#
#################################################################################################################################################
import webbrowser, requests, time, sys, bs4, os
print('Implementing the shell script using the Python requests module...')
print("Let us begin the retrieval of statistics Through SStats in the OpenSim server.")
print()
print("Retrieval is done every 1 second for 3 minutes or during 180 seconds (i.e 180 values intake)")
print()
print(" It is done through the retrival of HTML files from the HTTP Server (Web Statistics Module) of OpenSim thorough Python requests module...")
print(" on http://0.0.0.0:9000/SStats/")
print()
print()
print()
print("Please go and launch any avatar mobility script you want - you can not do anything here if you want, just press any key ...")
time.sleep(2)
print("Creating a folder named SStats_simstatsajax (if it does not exist)....")
if not os.path.exists('SStats_simstatsajax'):
os.makedirs('SStats_simstatsajax')
print("Creating a folder named SStats_activelogajax (if it does not exist)....")
if not os.path.exists('SStats_activelogajax'):
os.makedirs('SStats_activelogajax')
print("Creating a folder named SStats_activeconnectionsajax (if it does not exist)....")
if not os.path.exists('SStats_activeconnectionsajax'):
os.makedirs('SStats_activeconnectionsajax')
print('\n..............................................')
print()
for i in range(0,180):
print(i)
#if request succeed, the downloaded web page is stored as a string in the Response object res text variable
res = requests.get('http://0.0.0.0:9000/SStats/simstatsajax.html')
res.raise_for_status()
print('\nLength of the response: ' + str(len(res.text)))
HTMLFile = open('SStats_simstatsajax/%s.html' % (i) , 'wb') #wb: for write binary mode-this to maintain the unicode encoding of the text
for chunck in res.iter_content(int(len(res.text))):
HTMLFile.write(chunck)
time.sleep(1)
#check if the files needs to be closed! maybe not
#print(type(res))
#print(res.status_code == requests.codes.ok)
#print(len(res.text))
#print(res.text[:90000])