analysis.py

"""
This script generates basic stats and a chart from all_responses.json.
The all_responses.json file should be generated by running the beam pipeline
in stack-overflow.py.

Usage: python3 analysis.py
"""

import re
import pandas as pd
import matplotlib.pyplot as plt

pd.set_option('display.max_columns', None)
pd.set_option('expand_frame_repr', False)


def classify_error(row):
    if row["err_type"] == "HTTPError":
        return "HTTP_{}".format(row["status"])
    elif "Read timed out" in row["err_details"]:
        return "READ_TIMEOUT"
    elif "Name or service not known" in row["err_details"]:
        return "DNS_ERROR"
    elif "No address associated with hostname" in row["err_details"]:
        return "DNS_ERROR"
    elif "Temporary failure in name resolution" in row["err_details"]:
        return "DNS_ERROR"
    elif "Connection refused" in row["err_details"]:
        return "CONN_REFUSED"
    elif "Connection reset by peer" in row["err_details"]:
        return "CONN_RESET"
    elif "SSLError" in row["err_details"]:
        return "SSL_ERROR"
    elif "Network is unreachable" in row["err_details"]:
        return "UNREACHABLE"
    elif "No route to host" in row["err_details"]:
        return "NO_ROUTE"
    elif "Remote end closed connection without response" in row["err_details"]:
        return "CONN_CLOSED"
    elif re.search(r"Connection to [\w\.\-]+ timed out", row["err_details"]) is not None:
        return "CONN_TIMEOUT"
    else:
        raise Exception("Unclassified error {}".format(row["err_details"]))


def main():
    df = pd.read_json("./all_responses.json")
    errors = df[~df["err_type"].isnull()]
    num_errors = len(errors)
    num_responses = len(df)
    print("Out of {} URLs checked, {} returned errors. ({:.2f}%)".format(num_responses, num_errors,
                                                                         100 * num_errors / num_responses))

    error_types = errors.apply(classify_error, axis=1)
    plt.style.use('ggplot')
    plt.subplots_adjust(bottom=0.15)
    error_types.value_counts().plot(kind="bar", style="ggplot", ylabel="count",
                                    title="Stack Overflow Broken URL Causes")
    plt.subplots_adjust(bottom=0.15)
    plt.show()


if __name__ == "__main__":
    main()