diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 90306d2..c4876c5 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -9,10 +9,15 @@ import sys from collections import defaultdict import os +report_untrused=False + cipherstats = defaultdict(int) pfsstats = defaultdict(int) protocolstats = defaultdict(int) handshakestats = defaultdict(int) +keysize = defaultdict(int) +sigalg = defaultdict(int) +dsarsastack = 0 total = 0 for r,d,flist in os.walk(path): @@ -20,9 +25,14 @@ for r,d,flist in os.walk(path): """ initialize variables for stats of the current site """ temppfsstats = {} + tempkeystats = {} + tempecckeystats = {} + tempdsakeystats = {} + tempsigstats = {} ciphertypes = 0 AESGCM = False AES = False + CHACHA20 = False DES3 = False CAMELLIA = False RC4 = False @@ -34,6 +44,9 @@ for r,d,flist in os.walk(path): TLS1 = False TLS1_1 = False TLS1_2 = False + dualstack = False + ECDSA = False + trusted = False """ process the file """ f_abs = os.path.join(r,f) @@ -48,13 +61,17 @@ for r,d,flist in os.walk(path): if len(results['ciphersuite']) < 1: continue - total += 1 - """ loop over list of ciphers """ for entry in results['ciphersuite']: + # some servers return different certificates with different + # ciphers, also we may become redirected to other server with + # different config (because over-reactive IPS) + if 'False' in entry['trusted'] and report_untrused == False: + continue + """ store the ciphers supported """ - if 'AES-GCM' in entry['cipher']: + if 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']: if not AESGCM: AESGCM = True ciphertypes += 1 @@ -74,6 +91,10 @@ for r,d,flist in os.walk(path): if not RC4: ciphertypes += 1 RC4 = True + elif 'CHACHA20' in entry['cipher']: + if not CHACHA20: + ciphertypes += 1 + CHACHA20 = True else: ciphertypes += 1 name = "z:" + entry['cipher'] @@ -87,6 +108,25 @@ for r,d,flist in os.walk(path): DHE = True temppfsstats[entry['pfs']] = 1 + """ save the key size """ + if 'ECDSA' in entry['cipher']: + ECDSA = True + tempecckeystats[entry['pubkey'][0]] = 1 + elif 'DSS' in entry['cipher']: + tempdsakeystats[entry['pubkey'][0]] = 1 + elif 'AECDH' in entry['cipher'] or 'ADH' in entry['cipher']: + """ skip """ + else: + tempkeystats[entry['pubkey'][0]] = 1 + if ECDSA: + dualstack = True + + if 'True' in entry['trusted'] and not 'ADH' in entry['cipher'] and not 'AECDH' in entry['cipher']: + trusted = True + + """ save key signatures size """ + tempsigstats[entry['sigalg'][0]] = 1 + """ store the versions of TLS supported """ for protocol in entry['protocols']: if protocol == 'SSLv2': @@ -101,14 +141,34 @@ for r,d,flist in os.walk(path): TLS1_2 = True json_file.close() + """ don't store stats from unusued servers """ + if report_untrused == False and trusted == False: + continue + + total += 1 + """ done with this file, storing the stats """ if DHE or ECDHE: pfsstats['Support PFS'] += 1 if 'DHE-' in results['ciphersuite'][0]['cipher']: pfsstats['Prefer PFS'] += 1 + pfsstats['Prefer ' + results['ciphersuite'][0]['pfs']] += 1 for s in temppfsstats: pfsstats[s] += 1 + for s in tempkeystats: + keysize['RSA ' + s] += 1 + for s in tempecckeystats: + keysize['ECDSA ' + s] += 1 + for s in tempdsakeystats: + keysize['DSA ' + s] += 1 + + if dualstack: + dsarsastack += 1 + + for s in tempsigstats: + sigalg[s] += 1 + """ store cipher stats """ if AESGCM: cipherstats['AES-GCM'] += 1 @@ -117,7 +177,14 @@ for r,d,flist in os.walk(path): if AES: cipherstats['AES'] += 1 if ciphertypes == 1: + cipherstats['AES-CBC Only'] += 1 + if (AES and ciphertypes == 1) or (AESGCM and ciphertypes == 1)\ + or (AES and AESGCM and ciphertypes == 2): cipherstats['AES Only'] += 1 + if CHACHA20: + cipherstats['CHACHA20'] += 1 + if ciphertypes == 1: + cipherstats['CHACHA20 Only'] += 1 if DES3: cipherstats['3DES'] += 1 if ciphertypes == 1: @@ -130,6 +197,12 @@ for r,d,flist in os.walk(path): cipherstats['RC4'] += 1 if ciphertypes == 1: cipherstats['RC4 Only'] += 1 + if 'RC4' in results['ciphersuite'][0]['cipher']: + if 'TLSv1.1' in results['ciphersuite'][0]['protocols'] or\ + 'TLSv1.2' in results['ciphersuite'][0]['protocols']: + cipherstats['RC4 forced in TLS1.1+'] += 1 + cipherstats['RC4 Preferred'] += 1 + """ store handshake stats """ if ECDHE: @@ -152,22 +225,30 @@ for r,d,flist in os.walk(path): protocolstats['TLS1'] += 1 if not SSL2 and not SSL3 and not TLS1_1 and not TLS1_2: protocolstats['TLS1 Only'] += 1 + if not SSL2 and (SSL3 or TLS1) and not TLS1_1 and not TLS1_2: + protocolstats['SSL3 or TLS1 Only'] += 1 + if not SSL2 and not SSL3 and not TLS1: + protocolstats['TLS1.1 or up Only'] += 1 if TLS1_1: protocolstats['TLS1.1'] += 1 if not SSL2 and not SSL3 and not TLS1 and not TLS1_2: - protocolstats['TLS1_1 Only'] += 1 + protocolstats['TLS1.1 Only'] += 1 if TLS1_2: protocolstats['TLS1.2'] += 1 if not SSL2 and not SSL3 and not TLS1 and not TLS1_1: protocolstats['TLS1.2 Only'] += 1 - if TLS1_2 and not TLS1_1: - protocolstats['TLS1.2 but not 1.1'] += 1 + if TLS1_2 and not TLS1_1 and TLS1: + protocolstats['TLS1.2, 1.0 but not 1.1'] += 1 # for testing, break early #if total % 1999 == 0: # break print("SSL/TLS survey of %i websites from Alexa's top 1 million" % total) +if report_untrused == False: + print("Stats only from connections that did provide valid certificates") + print("(or anonymous DH from servers that do also have valid certificate installed)\n") + """ Display stats """ print("\nSupported Ciphers Count Percent") print("-------------------------+---------+-------") @@ -192,6 +273,20 @@ for stat in sorted(pfsstats): pfspercent = round(pfsstats[stat] / handshakestats['DHE'] * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(pfsstats[stat]).ljust(10) + str(percent).ljust(9) + str(pfspercent) + "\n") +print("\nCertificate sig alg Count Percent ") +print("-------------------------+---------+--------") +for stat in sorted(sigalg): + percent = round(sigalg[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(sigalg[stat]).ljust(10) + str(percent).ljust(9) + "\n") + +print("\nCertificate key size Count Percent ") +print("-------------------------+---------+--------") +for stat in sorted(keysize): + percent = round(keysize[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + str(percent).ljust(9) + "\n") + +sys.stdout.write("RSA/ECDSA Dual Stack".ljust(25) + " " + str(dsarsastack).ljust(10) + str(round(dsarsastack/total * 100, 4)) + "\n") + print("\nSupported Protocols Count Percent") print("-------------------------+---------+-------") for stat in sorted(protocolstats):