diff --git a/ParseHTMLToCSVsimstatsajax.py b/ParseHTMLToCSVsimstatsajax.py new file mode 100644 index 0000000..b1ca7f2 --- /dev/null +++ b/ParseHTMLToCSVsimstatsajax.py @@ -0,0 +1,120 @@ +#!/bin/python3 +################################################################################################################################################# +#Author: Hussein Bakri +#Script Title: Parses HTML files from Web Statistics AJAX module - simstatsajax- into ONE CSV file for later statistical analysis. +#License: GNU GPL v3 License - you are free to distribute, change, enhance and include any of the code of this script in your tools. I only expect #adequate attribution of this work. The attribution should include the title of the script, the author and the site or the document where the #script is taken from. +#Python 3 is needed +#This script Parses the HTML files obtained and stored in SStats_simstatsajax and transform all of them into one CSV file for statistical analysis +#It utilizes the BeautifulSoup Python module (bs4) which needs to be installed (on Linux: sudo pip3 install BeautifulSoup) +#It utilizes also the csv module which needs to be available +#-----IN EACH HTML - the 'td' HTML tag is what important to get--------------- +#By assigning the following: +#table = SoupObj.find('table') +#rows = table.findAll('tr') +#cols = table.findAll('td') +#from colum 0 till 9 correspond from Dilatn to ScrLPS +#colum 10 till 19 correspond to values under them +#from colum 20 till 28 correspond from Dilatn to ScrLPS +#colum 29 till 37 correspond to values under them +# +################################################################################################################################################# +import bs4, csv, time +import itertools as it + +print('Parsing the HTMLs and storing them into one CSV: output.csv...') +print() +#time.sleep(2) +#creating a CSV file and writing a header row +print('Creating the CSV file named: output.csv') +outputfile = open('output.csv', 'w', newline='') +outputWriter = csv.writer(outputfile) +print() +print('\n......................................................') +print() + +print('Processing first HTML file which should be 0.html, please wait...') +loadedHTMLFile = open('0.html') +SoupObj = bs4.BeautifulSoup(loadedHTMLFile.read()) +Alltables = SoupObj.findAll('table') +rowsOfFirstTable = Alltables[0].findAll('tr') +colsOfFirstTable = Alltables[0].findAll('td') +RegionNames = SoupObj.findAll('h2') +print() +print() + + +#Writing the header into header list +header=[] +print(range(0,len(RegionNames))) +for j in range(0,len(RegionNames)): + for i in it.chain(range(0, 10), range(20, 29)): + #Writing the header columns names + print(colsOfFirstTable[i].string) # Write this to CSV + header.append(RegionNames[j].getText() + ' ' + colsOfFirstTable[i].string) +#Writing the header file of the CSV +print('Writing the header of the CSV file, please wait....') +outputWriter.writerow(header) + +print('Fetching the values of this header, please wait...') +valuesRow1=[] +for j in range(0,len(RegionNames)): + for i in it.chain(range(10, 20), range(29, 38)): + #Writing the values of this HTML under exactly each colum + CurrentCols = Alltables[j].findAll('td') + print(CurrentCols[i].string) # Write this to CSV + valuesRow1.append(CurrentCols[i].string) + +#Writing the values to the CSV +print('Writing the values to the CSV file, please wait....') +outputWriter.writerow(valuesRow1) + + +print('Storing other rows from other HTML files into the CSV. Please wait....') + +for files in range(1,180): # 3+ time of inner loop in seconds + loadedHTMLFile = open('%s.html' % (files)) + print('Storing content of:' + '%s.html' % (files)) + time.sleep(1) + SoupObj = bs4.BeautifulSoup(loadedHTMLFile.read()) + valuesRow=[] + Alltables = SoupObj.findAll('table') + #colsOfFirstTable = Alltables[0].findAll('td') + time.sleep(1) + for j in range(0,len(RegionNames)): + for i in it.chain(range(10, 20), range(29, 38)): + #Writing the values of this HTML under exactly each colum + CurrentCols = Alltables[j].findAll('td') + print(CurrentCols[i].string) # Write this to CSV + valuesRow.append(CurrentCols[i].string) + print(valuesRow) # Write this to CSV + outputWriter.writerow(valuesRow) + time.sleep(1) + +print('Done') +loadedHTMLFile.close() +outputfile.close() + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SStatscript b/SStatscript new file mode 100644 index 0000000..6e93fd7 --- /dev/null +++ b/SStatscript @@ -0,0 +1,61 @@ +#!/bin/bash +################################################################################################################################################## +#Author: Hussein Bakri +#Script Title: Method of getting stats from the OpenSim AJAX Web Statistical module and storing them in HTML files +#License: GNU GPL v3 License - you are free to distribute, change, enhance and include any of the code of this script in your tools. I only expect #adequate attribution of this work. The attribution should include the title of the script, the author and the site or the document where the #script is taken from. +#The Web Statistics Module provides region statistics information. The data is provided as AJAX html page which automatically updates over time. #The page is provided by the internal OpenSimulator web server. The module also stores historical data which is displayed on these web pages. +#----------- Retrieve HTML files from Web Statistics AJAX module -simstatsajax/activelogajax/activeconnectionsajax--------- +#This Bash shell script saves statistics every 1 second for 3 minutes (during 180 seconds) from the OpenSim Web Statistics Module +#OpenSim Wiki article : http://opensimulator.org/wiki/Web_Statistics_Module +#The challenge solved is to get the XHR calls URLs of the AJAX page - This was done through using the Developer tools of Web browsers mainly the +#Network Inspectors. The 3 XHR call URLS are being fetched regularly on the AJAX page: +#http://0.0.0.0:9000/SStats/simstatsajax.html - have useful Simulator stats +#http://0.0.0.0:9000/SStats/activelogajax.html - NOTHING USEFUL HERE. It is retrieved for completion sake +#http://0.0.0.0:9000/SStats/activeconnectionsajax.html - has useful stats about packets in the different channels of circuits of each region. +#The Shell script fetches XHR URLs using wget command and stores the output as HTML files in seperate folders (3 folders) +#Python script(s) will later -in each folder- parse the HTML files and store their content into ONE CSV file for later statistical Analysis. +#This script will generate many files and take some time.... +################################################################################################################################################## +echo "Let us begin the retrieval of statistics Through Web Statistics Module in the OpenSim server..." +echo +echo "Retrieval is done every 1 second for 3 minutes or during 180 seconds (i.e 180 values intake)" +echo +echo " It is done through the retrival of HTML files from the HTTP Server (Web Statistics Module) of OpenSim through the wget command..." +echo " on http://:9000/SStats/" + +echo +echo +echo + +echo "Please go and launch any avatar mobility script you want - you can not do anything here if you want, just press any key ..." +sleep 2 +read -rsp $'When ready, Press any key in this terminal to continue - terminal should be in focus...\n' -n1 key + +echo "Creating a folder named SStats_simstatsajax (if it does not exist)...." +mkdir -p SStats_simstatsajax +echo + +echo "Creating a folder named SStats_activelogajax (if it does not exist)...." +mkdir -p SStats_activelogajax +echo + +echo "Creating a folder named SStats_activeconnectionsajax (if it does not exist)...." +mkdir -p SStats_activeconnectionsajax +echo + +filenumber=0 +while [ $filenumber -lt 180 ] +do +wget http://0.0.0.0:9000/SStats/simstatsajax.html -O SStats_simstatsajax/${filenumber}.html +wget http://0.0.0.0:9000/SStats/activelogajax.html -O SStats_activelogajax/${filenumber}.html +wget http://0.0.0.0:9000/SStats/activeconnectionsajax.html -O SStats_activeconnectionsajax/${filenumber}.html +sleep 1 +((filenumber++)) +done +echo "Retrieval Finished ...." +echo +echo "..." + +echo "Bye" + + diff --git a/requestsTest.py b/requestsTest.py new file mode 100644 index 0000000..57545c5 --- /dev/null +++ b/requestsTest.py @@ -0,0 +1,85 @@ +#!/bin/python3 +################################################################################################################################################# +#Author: Hussein Bakri +#Script Title: Retrieve HTML files from Web Statistics AJAX module - simstatsajax +#License: GNU GPL v3 License - you are free to distribute, change, enhance and include any of the code of this script in your tools. I only expect #adequate attribution of this work. The attribution should include the title of the script, the author and the site or the document where the #script is taken from. +#---------- +# +#Python 3 is needed +#This script transform Transform JSON files that Web Statistics AJAX module through UXSimStatus into ONE CSV file for later statistical analysis +# It uses the Request module to request every 1 second for 3 minutes (180 seconds) http://0.0.0.0:9000/SStats/simstatsajax.html' +# It Implements another shell script named: "SStatscript" using the Python requests module +# +################################################################################################################################################# +import webbrowser, requests, time, sys, bs4, os + +print('Implementing the shell script using the Python requests module...') +print("Let us begin the retrieval of statistics Through SStats in the OpenSim server.") +print() +print("Retrieval is done every 1 second for 3 minutes or during 180 seconds (i.e 180 values intake)") +print() +print(" It is done through the retrival of HTML files from the HTTP Server (Web Statistics Module) of OpenSim thorough Python requests module...") +print(" on http://0.0.0.0:9000/SStats/") + +print() +print() +print() + +print("Please go and launch any avatar mobility script you want - you can not do anything here if you want, just press any key ...") +time.sleep(2) + +print("Creating a folder named SStats_simstatsajax (if it does not exist)....") +if not os.path.exists('SStats_simstatsajax'): + os.makedirs('SStats_simstatsajax') + + +print("Creating a folder named SStats_activelogajax (if it does not exist)....") +if not os.path.exists('SStats_activelogajax'): + os.makedirs('SStats_activelogajax') + +print("Creating a folder named SStats_activeconnectionsajax (if it does not exist)....") +if not os.path.exists('SStats_activeconnectionsajax'): + os.makedirs('SStats_activeconnectionsajax') + +print('\n..............................................') +print() + +for i in range(0,180): + print(i) + #if request succeed, the downloaded web page is stored as a string in the Response object res text variable + res = requests.get('http://0.0.0.0:9000/SStats/simstatsajax.html') + res.raise_for_status() + print('\nLength of the response: ' + str(len(res.text))) + HTMLFile = open('SStats_simstatsajax/%s.html' % (i) , 'wb') #wb: for write binary mode-this to maintain the unicode encoding of the text + for chunck in res.iter_content(int(len(res.text))): + HTMLFile.write(chunck) + time.sleep(1) + +#check if the files needs to be closed! maybe not +#print(type(res)) +#print(res.status_code == requests.codes.ok) +#print(len(res.text)) +#print(res.text[:90000]) + + + + + + + + + + + + + + + + + + + + + + +