diff --git a/top1m/parse_results.py b/top1m/parse_results.py new file mode 100644 index 0000000..90306d2 --- /dev/null +++ b/top1m/parse_results.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python + +from __future__ import division + +path = "./results/" + +import json +import sys +from collections import defaultdict +import os + +cipherstats = defaultdict(int) +pfsstats = defaultdict(int) +protocolstats = defaultdict(int) +handshakestats = defaultdict(int) +total = 0 +for r,d,flist in os.walk(path): + + for f in flist: + + """ initialize variables for stats of the current site """ + temppfsstats = {} + ciphertypes = 0 + AESGCM = False + AES = False + DES3 = False + CAMELLIA = False + RC4 = False + DHE = False + ECDHE = False + RSA = False + SSL2 = False + SSL3 = False + TLS1 = False + TLS1_1 = False + TLS1_2 = False + + """ process the file """ + f_abs = os.path.join(r,f) + with open(f_abs) as json_file: + """ discard files that fail to load """ + try: + results = json.load(json_file) + except ValueError: + continue + + """ discard files with empty results """ + if len(results['ciphersuite']) < 1: + continue + + total += 1 + + """ loop over list of ciphers """ + for entry in results['ciphersuite']: + + """ store the ciphers supported """ + if 'AES-GCM' in entry['cipher']: + if not AESGCM: + AESGCM = True + ciphertypes += 1 + elif 'AES' in entry['cipher']: + if not AES: + AES = True + ciphertypes += 1 + elif 'DES-CBC3' in entry['cipher']: + if not DES3: + DES3 = True + ciphertypes += 1 + elif 'CAMELLIA' in entry['cipher']: + if not CAMELLIA: + CAMELLIA = True + ciphertypes += 1 + elif 'RC4' in entry['cipher']: + if not RC4: + ciphertypes += 1 + RC4 = True + else: + ciphertypes += 1 + name = "z:" + entry['cipher'] + cipherstats[name] += 1 + + """ store key handshake methods """ + if 'ECDHE' in entry['cipher']: + ECDHE = True + temppfsstats[entry['pfs']] = 1 + elif 'DHE' in entry['cipher']: + DHE = True + temppfsstats[entry['pfs']] = 1 + + """ store the versions of TLS supported """ + for protocol in entry['protocols']: + if protocol == 'SSLv2': + SSL2 = True + elif protocol == 'SSLv3': + SSL3 = True + elif protocol == 'TLSv1': + TLS1 = True + elif protocol == 'TLSv1.1': + TLS1_1 = True + elif protocol == 'TLSv1.2': + TLS1_2 = True + json_file.close() + + """ done with this file, storing the stats """ + if DHE or ECDHE: + pfsstats['Support PFS'] += 1 + if 'DHE-' in results['ciphersuite'][0]['cipher']: + pfsstats['Prefer PFS'] += 1 + for s in temppfsstats: + pfsstats[s] += 1 + + """ store cipher stats """ + if AESGCM: + cipherstats['AES-GCM'] += 1 + if ciphertypes == 1: + cipherstats['AES-GCM Only'] += 1 + if AES: + cipherstats['AES'] += 1 + if ciphertypes == 1: + cipherstats['AES Only'] += 1 + if DES3: + cipherstats['3DES'] += 1 + if ciphertypes == 1: + cipherstats['3DES Only'] += 1 + if CAMELLIA: + cipherstats['CAMELLIA'] += 1 + if ciphertypes == 1: + cipherstats['CAMELLIA Only'] += 1 + if RC4: + cipherstats['RC4'] += 1 + if ciphertypes == 1: + cipherstats['RC4 Only'] += 1 + + """ store handshake stats """ + if ECDHE: + handshakestats['ECDHE'] += 1 + if DHE: + handshakestats['DHE'] += 1 + if RSA: + handshakestats['RSA'] += 1 + + """ store protocol stats """ + if SSL2: + protocolstats['SSL2'] += 1 + if not SSL3 and not TLS1 and not TLS1_1 and not TLS1_2: + protocolstats['SSL2 Only'] += 1 + if SSL3: + protocolstats['SSL3'] += 1 + if not SSL2 and not TLS1 and not TLS1_1 and not TLS1_2: + protocolstats['SSL3 Only'] += 1 + if TLS1: + protocolstats['TLS1'] += 1 + if not SSL2 and not SSL3 and not TLS1_1 and not TLS1_2: + protocolstats['TLS1 Only'] += 1 + if TLS1_1: + protocolstats['TLS1.1'] += 1 + if not SSL2 and not SSL3 and not TLS1 and not TLS1_2: + protocolstats['TLS1_1 Only'] += 1 + if TLS1_2: + protocolstats['TLS1.2'] += 1 + if not SSL2 and not SSL3 and not TLS1 and not TLS1_1: + protocolstats['TLS1.2 Only'] += 1 + if TLS1_2 and not TLS1_1: + protocolstats['TLS1.2 but not 1.1'] += 1 + + # for testing, break early + #if total % 1999 == 0: + # break + +print("SSL/TLS survey of %i websites from Alexa's top 1 million" % total) +""" Display stats """ +print("\nSupported Ciphers Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(cipherstats): + percent = round(cipherstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nSupported Handshakes Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(handshakestats): + percent = round(handshakestats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(handshakestats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nSupported PFS Count Percent PFS Percent") +print("-------------------------+---------+--------+-----------") +for stat in sorted(pfsstats): + percent = round(pfsstats[stat] / total * 100, 4) + pfspercent = 0 + if "ECDH," in stat: + pfspercent = round(pfsstats[stat] / handshakestats['ECDHE'] * 100, 4) + elif "DH," in stat: + pfspercent = round(pfsstats[stat] / handshakestats['DHE'] * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(pfsstats[stat]).ljust(10) + str(percent).ljust(9) + str(pfspercent) + "\n") + +print("\nSupported Protocols Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(protocolstats): + percent = round(protocolstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(protocolstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index 6c76816..52f504b 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -1,18 +1,14 @@ #!/usr/bin/env bash +parallel=50 [ ! -e "results" ] && mkdir results i=1 while [ $i -lt 1000000 ] do - echo processings sites $i to $((i + 50)) - for t in $(tail -$((1000000 - $i)) top-1m.csv | head -50 |cut -d ',' -f 2) + echo processings sites $i to $((i + parallel)) + for t in $(tail -$((1000000 - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2) do - tcping -u 2000000 $t 443 - if [ $? -gt 0 ] - then - continue - fi - ../cipherscan $t:443 > results/$t & + (tcping -u 10000000 $t 443; if [ $? -gt 0 ];then continue;fi;../cipherscan $t:443 -json > results/$t )& done - sleep 10 - i=$(( i + 50)) + sleep 7 + i=$(( i + parallel)) done