"""

Returns the confusion matrix between rater's ratings

"""

assert(len(rater_a) == len(rater_b))

if min_rating is None:

min_rating = min(rater_a + rater_b)

if max_rating is None:

max_rating = max(rater_a + rater_b)

num_ratings = int(max_rating - min_rating + 1)

conf_mat = [[0 for i in range(num_ratings)]

for j in range(num_ratings)]

for a, b in zip(rater_a, rater_b):

conf_mat[a - min_rating][b - min_rating] += 1

"""

Returns the counts of each type of rating that a rater made

"""

if min_rating is None:

min_rating = min(ratings)

if max_rating is None:

max_rating = max(ratings)

num_ratings = int(max_rating - min_rating + 1)

hist_ratings = [0 for x in range(num_ratings)]

for r in ratings:

hist_ratings[r - min_rating] += 1

"""

Calculates the quadratic weighted kappa

quadratic_weighted_kappa calculates the quadratic weighted kappa

value, which is a measure of inter-rater agreement between two raters

that provide discrete numeric ratings. Potential values range from -1

(representing complete disagreement) to 1 (representing complete

agreement). A kappa value of 0 is expected if all agreement is due to

chance.

quadratic_weighted_kappa(rater_a, rater_b), where rater_a and rater_b

each correspond to a list of integer ratings. These lists must have the

same length.

The ratings should be integers, and it is assumed that they contain

the complete range of possible ratings.

quadratic_weighted_kappa(X, min_rating, max_rating), where min_rating

is the minimum possible rating, and max_rating is the maximum possible

rating

"""

rater_a = np.array(rater_a, dtype=int)

rater_b = np.array(rater_b, dtype=int)

assert(len(rater_a) == len(rater_b))

if min_rating is None:

min_rating = min(min(rater_a), min(rater_b))

if max_rating is None:

max_rating = max(max(rater_a), max(rater_b))

conf_mat = confusion_matrix(rater_a, rater_b,

min_rating, max_rating)

num_ratings = len(conf_mat)

num_scored_items = float(len(rater_a))

hist_rater_a = histogram(rater_a, min_rating, max_rating)

hist_rater_b = histogram(rater_b, min_rating, max_rating)

numerator = 0.0

denominator = 0.0

for i in range(num_ratings):

for j in range(num_ratings):

expected_count = (hist_rater_a[i] * hist_rater_b[j]

/ num_scored_items)

d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)

numerator += d * conf_mat[i][j] / num_scored_items

denominator += d * expected_count / num_scored_items

"""

Calculates the linear weighted kappa

linear_weighted_kappa calculates the linear weighted kappa

value, which is a measure of inter-rater agreement between two raters

that provide discrete numeric ratings. Potential values range from -1

(representing complete disagreement) to 1 (representing complete

agreement). A kappa value of 0 is expected if all agreement is due to

chance.

linear_weighted_kappa(rater_a, rater_b), where rater_a and rater_b

each correspond to a list of integer ratings. These lists must have the

same length.

The ratings should be integers, and it is assumed that they contain

the complete range of possible ratings.

linear_weighted_kappa(X, min_rating, max_rating), where min_rating

is the minimum possible rating, and max_rating is the maximum possible

rating

"""

assert(len(rater_a) == len(rater_b))

if min_rating is None:

min_rating = min(rater_a + rater_b)

if max_rating is None:

max_rating = max(rater_a + rater_b)

conf_mat = confusion_matrix(rater_a, rater_b,

min_rating, max_rating)

num_ratings = len(conf_mat)

num_scored_items = float(len(rater_a))

hist_rater_a = histogram(rater_a, min_rating, max_rating)

hist_rater_b = histogram(rater_b, min_rating, max_rating)

numerator = 0.0

denominator = 0.0

for i in range(num_ratings):

for j in range(num_ratings):

expected_count = (hist_rater_a[i] * hist_rater_b[j]

/ num_scored_items)

d = abs(i - j) / float(num_ratings - 1)

numerator += d * conf_mat[i][j] / num_scored_items

denominator += d * expected_count / num_scored_items

"""

Calculates the kappa

kappa calculates the kappa

value, which is a measure of inter-rater agreement between two raters

that provide discrete numeric ratings. Potential values range from -1

(representing complete disagreement) to 1 (representing complete

agreement). A kappa value of 0 is expected if all agreement is due to

chance.

kappa(rater_a, rater_b), where rater_a and rater_b

each correspond to a list of integer ratings. These lists must have the

same length.

The ratings should be integers, and it is assumed that they contain

the complete range of possible ratings.

kappa(X, min_rating, max_rating), where min_rating

is the minimum possible rating, and max_rating is the maximum possible

rating

"""

assert(len(rater_a) == len(rater_b))

if min_rating is None:

min_rating = min(rater_a + rater_b)

if max_rating is None:

max_rating = max(rater_a + rater_b)

conf_mat = confusion_matrix(rater_a, rater_b,

min_rating, max_rating)

num_ratings = len(conf_mat)

num_scored_items = float(len(rater_a))

hist_rater_a = histogram(rater_a, min_rating, max_rating)

hist_rater_b = histogram(rater_b, min_rating, max_rating)

numerator = 0.0

denominator = 0.0

for i in range(num_ratings):

for j in range(num_ratings):

expected_count = (hist_rater_a[i] * hist_rater_b[j]

/ num_scored_items)

if i == j:

d = 0.0

else:

d = 1.0

numerator += d * conf_mat[i][j] / num_scored_items

denominator += d * expected_count / num_scored_items

"""

Calculates the mean of the quadratic

weighted kappas after applying Fisher's r-to-z transform, which is

approximately a variance-stabilizing transformation. This

transformation is undefined if one of the kappas is 1.0, so all kappa

values are capped in the range (-0.999, 0.999). The reverse

transformation is then applied before returning the result.

mean_quadratic_weighted_kappa(kappas), where kappas is a vector of

kappa values

mean_quadratic_weighted_kappa(kappas, weights), where weights is a vector

of weights that is the same size as kappas. Weights are applied in the

z-space

"""

kappas = np.array(kappas, dtype=float)

if weights is None:

weights = np.ones(np.shape(kappas))

else:

weights = weights / np.mean(weights)

# ensure that kappas are in the range [-.999, .999]

kappas = np.array([min(x, .999) for x in kappas])

kappas = np.array([max(x, -.999) for x in kappas])

z = 0.5 * np.log((1 + kappas) / (1 - kappas)) * weights

z = np.mean(z)

predicted_score = submission[submission.columns[-1]].copy()

predicted_score.name = "predicted_score"

if predicted_score.index[0] == 0:

predicted_score = predicted_score[:len(solution)]

predicted_score.index = solution.index

combined = solution.join(predicted_score, how="left")

groups = combined.groupby(by="essay_set")

kappas = [quadratic_weighted_kappa(group[1]["essay_score"], group[1]["predicted_score"]) for group in groups]

weights = [group[1]["essay_weight"].irow(0) for group in groups]