From f3c8b24b8ba4da7e6615dc82e65c6ccebda7f31e Mon Sep 17 00:00:00 2001 From: Julien Vehent Date: Thu, 9 Jan 2014 20:16:40 +0000 Subject: [PATCH 1/3] tweaks --- top1m/testtop1m.sh | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index 6c76816..52f504b 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -1,18 +1,14 @@ #!/usr/bin/env bash +parallel=50 [ ! -e "results" ] && mkdir results i=1 while [ $i -lt 1000000 ] do - echo processings sites $i to $((i + 50)) - for t in $(tail -$((1000000 - $i)) top-1m.csv | head -50 |cut -d ',' -f 2) + echo processings sites $i to $((i + parallel)) + for t in $(tail -$((1000000 - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2) do - tcping -u 2000000 $t 443 - if [ $? -gt 0 ] - then - continue - fi - ../cipherscan $t:443 > results/$t & + (tcping -u 10000000 $t 443; if [ $? -gt 0 ];then continue;fi;../cipherscan $t:443 -json > results/$t )& done - sleep 10 - i=$(( i + 50)) + sleep 7 + i=$(( i + parallel)) done From 141497353183ce7cc1808fff63058c5b397ff72d Mon Sep 17 00:00:00 2001 From: Julien Vehent Date: Fri, 10 Jan 2014 05:50:03 +0000 Subject: [PATCH 2/3] basic results parsing script in python --- top1m/parse_results.py | 84 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 top1m/parse_results.py diff --git a/top1m/parse_results.py b/top1m/parse_results.py new file mode 100644 index 0000000..7223387 --- /dev/null +++ b/top1m/parse_results.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +path = "./results/" + +import json +import sys +from collections import defaultdict +import os + +stats = defaultdict(int) + +for r,d,flist in os.walk(path): + for f in flist: + f_abs = os.path.join(r,f) + with open(f_abs) as json_file: + AES = False + DESCBC3 = False + RC4SHA = False + RC4MD5 = False + ECDHE = False + GCM = False + SSL2 = False + SSL3 = False + TLS1 = False + TLS1_1 = False + TLS1_2 = False + stats['sites'] += 1 + results = json.load(json_file) + if len(results['ciphersuite']) < 1: + stats['broken'] += 1 + continue + for entry in results['ciphersuite']: + if 'AES' in entry['cipher']: + AES = True + if 'DES-CBC3' in entry['cipher']: + DESCBC3 = True + if 'RC4-SHA' == entry['cipher']: + RC4SHA = True + if 'RC4-MD5' == entry['cipher']: + RC4MD5 = True + if 'ECDHE' in entry['cipher']: + ECDHE = True + if 'GCM' in entry['cipher']: + GCM = True + for protocol in entry['protocols']: + if protocol == 'SSLv2': + SSL2 = True + if protocol == 'SSLv3': + SSL3 = True + if protocol == 'TLSv1': + TLS1 = True + if protocol == 'TLSv1.1': + TLS1_1 = True + if protocol == 'TLSv1.2': + TLS1_2 = True + if 'DHE' in results['ciphersuite'][0]['cipher']: + stats['PFS-FIRST'] += 1 + if AES: + stats['AES'] += 1 + if DESCBC3: + stats['DES-CBC3'] += 1 + if RC4SHA: + stats['RC4-SHA'] += 1 + if RC4MD5: + stats['RC4-MD5'] += 1 + if ECDHE: + stats['ECDHE'] += 1 + if GCM: + stats['AES-GCM'] += 1 + if not AES and not DESCBC3 and (RC4SHA or RC4MD5): + stats['RC4-ONLY'] += 1 + if SSL2: + stats['SSL2'] += 1 + if SSL3: + stats['SSL3'] += 1 + if TLS1: + stats['TLS1'] += 1 + if TLS1_1: + stats['TLS1_1'] += 1 + if TLS1_2: + stats['TLS1_2'] += 1 + json_file.close() + if stats['sites'] % 2000 == 0: + print stats From 5e8b495a18e5711af24a145c0a6cf7cf0c878a1b Mon Sep 17 00:00:00 2001 From: Julien Vehent Date: Sat, 11 Jan 2014 01:07:32 +0000 Subject: [PATCH 3/3] added many tests --- top1m/parse_results.py | 233 ++++++++++++++++++++++++++++++----------- 1 file changed, 174 insertions(+), 59 deletions(-) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 7223387..90306d2 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +from __future__ import division + path = "./results/" import json @@ -7,78 +9,191 @@ import sys from collections import defaultdict import os -stats = defaultdict(int) - +cipherstats = defaultdict(int) +pfsstats = defaultdict(int) +protocolstats = defaultdict(int) +handshakestats = defaultdict(int) +total = 0 for r,d,flist in os.walk(path): + for f in flist: + + """ initialize variables for stats of the current site """ + temppfsstats = {} + ciphertypes = 0 + AESGCM = False + AES = False + DES3 = False + CAMELLIA = False + RC4 = False + DHE = False + ECDHE = False + RSA = False + SSL2 = False + SSL3 = False + TLS1 = False + TLS1_1 = False + TLS1_2 = False + + """ process the file """ f_abs = os.path.join(r,f) with open(f_abs) as json_file: - AES = False - DESCBC3 = False - RC4SHA = False - RC4MD5 = False - ECDHE = False - GCM = False - SSL2 = False - SSL3 = False - TLS1 = False - TLS1_1 = False - TLS1_2 = False - stats['sites'] += 1 - results = json.load(json_file) - if len(results['ciphersuite']) < 1: - stats['broken'] += 1 + """ discard files that fail to load """ + try: + results = json.load(json_file) + except ValueError: continue + + """ discard files with empty results """ + if len(results['ciphersuite']) < 1: + continue + + total += 1 + + """ loop over list of ciphers """ for entry in results['ciphersuite']: - if 'AES' in entry['cipher']: - AES = True - if 'DES-CBC3' in entry['cipher']: - DESCBC3 = True - if 'RC4-SHA' == entry['cipher']: - RC4SHA = True - if 'RC4-MD5' == entry['cipher']: - RC4MD5 = True + + """ store the ciphers supported """ + if 'AES-GCM' in entry['cipher']: + if not AESGCM: + AESGCM = True + ciphertypes += 1 + elif 'AES' in entry['cipher']: + if not AES: + AES = True + ciphertypes += 1 + elif 'DES-CBC3' in entry['cipher']: + if not DES3: + DES3 = True + ciphertypes += 1 + elif 'CAMELLIA' in entry['cipher']: + if not CAMELLIA: + CAMELLIA = True + ciphertypes += 1 + elif 'RC4' in entry['cipher']: + if not RC4: + ciphertypes += 1 + RC4 = True + else: + ciphertypes += 1 + name = "z:" + entry['cipher'] + cipherstats[name] += 1 + + """ store key handshake methods """ if 'ECDHE' in entry['cipher']: ECDHE = True - if 'GCM' in entry['cipher']: - GCM = True + temppfsstats[entry['pfs']] = 1 + elif 'DHE' in entry['cipher']: + DHE = True + temppfsstats[entry['pfs']] = 1 + + """ store the versions of TLS supported """ for protocol in entry['protocols']: if protocol == 'SSLv2': SSL2 = True - if protocol == 'SSLv3': + elif protocol == 'SSLv3': SSL3 = True - if protocol == 'TLSv1': + elif protocol == 'TLSv1': TLS1 = True - if protocol == 'TLSv1.1': + elif protocol == 'TLSv1.1': TLS1_1 = True - if protocol == 'TLSv1.2': + elif protocol == 'TLSv1.2': TLS1_2 = True - if 'DHE' in results['ciphersuite'][0]['cipher']: - stats['PFS-FIRST'] += 1 - if AES: - stats['AES'] += 1 - if DESCBC3: - stats['DES-CBC3'] += 1 - if RC4SHA: - stats['RC4-SHA'] += 1 - if RC4MD5: - stats['RC4-MD5'] += 1 - if ECDHE: - stats['ECDHE'] += 1 - if GCM: - stats['AES-GCM'] += 1 - if not AES and not DESCBC3 and (RC4SHA or RC4MD5): - stats['RC4-ONLY'] += 1 - if SSL2: - stats['SSL2'] += 1 - if SSL3: - stats['SSL3'] += 1 - if TLS1: - stats['TLS1'] += 1 - if TLS1_1: - stats['TLS1_1'] += 1 - if TLS1_2: - stats['TLS1_2'] += 1 json_file.close() - if stats['sites'] % 2000 == 0: - print stats + + """ done with this file, storing the stats """ + if DHE or ECDHE: + pfsstats['Support PFS'] += 1 + if 'DHE-' in results['ciphersuite'][0]['cipher']: + pfsstats['Prefer PFS'] += 1 + for s in temppfsstats: + pfsstats[s] += 1 + + """ store cipher stats """ + if AESGCM: + cipherstats['AES-GCM'] += 1 + if ciphertypes == 1: + cipherstats['AES-GCM Only'] += 1 + if AES: + cipherstats['AES'] += 1 + if ciphertypes == 1: + cipherstats['AES Only'] += 1 + if DES3: + cipherstats['3DES'] += 1 + if ciphertypes == 1: + cipherstats['3DES Only'] += 1 + if CAMELLIA: + cipherstats['CAMELLIA'] += 1 + if ciphertypes == 1: + cipherstats['CAMELLIA Only'] += 1 + if RC4: + cipherstats['RC4'] += 1 + if ciphertypes == 1: + cipherstats['RC4 Only'] += 1 + + """ store handshake stats """ + if ECDHE: + handshakestats['ECDHE'] += 1 + if DHE: + handshakestats['DHE'] += 1 + if RSA: + handshakestats['RSA'] += 1 + + """ store protocol stats """ + if SSL2: + protocolstats['SSL2'] += 1 + if not SSL3 and not TLS1 and not TLS1_1 and not TLS1_2: + protocolstats['SSL2 Only'] += 1 + if SSL3: + protocolstats['SSL3'] += 1 + if not SSL2 and not TLS1 and not TLS1_1 and not TLS1_2: + protocolstats['SSL3 Only'] += 1 + if TLS1: + protocolstats['TLS1'] += 1 + if not SSL2 and not SSL3 and not TLS1_1 and not TLS1_2: + protocolstats['TLS1 Only'] += 1 + if TLS1_1: + protocolstats['TLS1.1'] += 1 + if not SSL2 and not SSL3 and not TLS1 and not TLS1_2: + protocolstats['TLS1_1 Only'] += 1 + if TLS1_2: + protocolstats['TLS1.2'] += 1 + if not SSL2 and not SSL3 and not TLS1 and not TLS1_1: + protocolstats['TLS1.2 Only'] += 1 + if TLS1_2 and not TLS1_1: + protocolstats['TLS1.2 but not 1.1'] += 1 + + # for testing, break early + #if total % 1999 == 0: + # break + +print("SSL/TLS survey of %i websites from Alexa's top 1 million" % total) +""" Display stats """ +print("\nSupported Ciphers Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(cipherstats): + percent = round(cipherstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nSupported Handshakes Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(handshakestats): + percent = round(handshakestats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(handshakestats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nSupported PFS Count Percent PFS Percent") +print("-------------------------+---------+--------+-----------") +for stat in sorted(pfsstats): + percent = round(pfsstats[stat] / total * 100, 4) + pfspercent = 0 + if "ECDH," in stat: + pfspercent = round(pfsstats[stat] / handshakestats['ECDHE'] * 100, 4) + elif "DH," in stat: + pfspercent = round(pfsstats[stat] / handshakestats['DHE'] * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(pfsstats[stat]).ljust(10) + str(percent).ljust(9) + str(pfspercent) + "\n") + +print("\nSupported Protocols Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(protocolstats): + percent = round(protocolstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(protocolstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")