{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import pandas as pd\n",
    "import seaborn as sn\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['-'], ['analytical'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['analytical'], ['-'], ['-'], ['analytical'], ['analytical'], ['analytical'], ['analytical'], ['-'], ['-'], ['-'], ['-'], ['pp attachment'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['vague'], ['-'], ['vague'], ['-'], ['-'], ['-'], ['analytical', ' analytical'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['coordination'], ['coordination'], ['coordination'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['coordination'], ['-'], ['-'], ['coordination'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['analytical'], ['-'], ['-'], ['coordination'], ['-'], ['analytical'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['vague'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['analytical'], ['analytical'], ['-'], ['-'], ['vague'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['coordination'], ['analytical', ' analytical', ' analytical'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['coordination'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['analytical'], ['analytical'], ['vague'], ['-'], ['vague']]\n",
      "[['-'], ['analytical'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['pp attachment'], ['pp attachment'], ['coordination'], ['-'], ['-'], ['-'], ['analytical'], ['analytical'], ['analytical'], ['analytical'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['vague'], ['-'], ['vague'], ['-'], ['-'], ['-'], ['-', ' analytical'], ['-'], ['-'], ['coordination'], ['-'], ['-'], ['-'], ['coordination'], ['coordination'], ['coordination'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['coordination'], ['-'], ['-'], ['coordination'], ['-'], ['-'], ['-'], ['-'], ['pp attachment'], ['-'], ['-'], ['-'], ['analytical'], ['-'], ['-'], ['coordination'], ['-'], ['analytical'], ['pp attachment'], ['-'], ['-'], ['-'], ['vague'], ['vague'], ['-'], ['-'], ['-'], ['vague'], ['-'], ['-'], [''], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['analytical'], ['analytical'], ['-'], ['-'], ['vague'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['analytical', ' analytical', ' analytical'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['coordination'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['-'], ['analytical'], ['analytical'], ['vague'], ['-'], ['vague']]\n"
     ]
    }
   ],
   "source": [
    "with open('Evaluasi.csv', encoding=\"utf-8\") as csvfile: \n",
    "    next(csvfile)\n",
    "    rawEvaluasi = csv.reader(csvfile, delimiter=',') \n",
    "    hasilSistem = [] \n",
    "    hasilAhli = []\n",
    "    for row in rawEvaluasi:\n",
    "        hasilSistem.append(row[3].lower().split(','))\n",
    "        hasilAhli.append(row[4].lower().split(','))\n",
    "print(hasilSistem)\n",
    "print(hasilAhli)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hitungConfusionMatrix(jenisAmbiguitas):\n",
    "    countTP = 0\n",
    "    countFN = 0\n",
    "    countFP=0\n",
    "    countTN=0\n",
    "    for i in range(len(hasilSistem)):\n",
    "        for j in range(len(hasilSistem[i])):\n",
    "            if (hasilSistem[i][j]==jenisAmbiguitas) and (hasilAhli[i][j]==jenisAmbiguitas):\n",
    "                countTP+=1\n",
    "            elif(hasilSistem[i][j]==jenisAmbiguitas) and (hasilAhli[i][j]=='-'):\n",
    "                countFP+=1\n",
    "            elif(hasilSistem[i][j]=='-') and (hasilAhli[i][j]==jenisAmbiguitas):\n",
    "                countFN+=1\n",
    "            elif(hasilSistem[i][j]=='-') and (hasilAhli[i][j]=='-'):\n",
    "                countTN+=1\n",
    "    return countTP,countFP,countFN,countTN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataraw(countTP,countFP,countFN,countTN):\n",
    "    raw_data = {\n",
    "                'Sistem - Ahli':['Ambigu','Tidak Ambigu'],\n",
    "                'Ambigu':[countTP,countFN],\n",
    "                'Tidak Ambigu':[countFP,countTN]\n",
    "    }\n",
    "    df = pd.DataFrame(raw_data, columns=['Sistem - Ahli','Ambigu','Tidak Ambigu'])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hitungNilaiKappa(countTP,countFP,countFN,countTN):\n",
    "    probAnalisisSama = 0\n",
    "    probAnalisisBeda = 0\n",
    "    probAmbigu = 0\n",
    "    probTidakAmbigu = 0\n",
    "    nilaiKappa = 0\n",
    "    totalAnalisis = countTP+countFP+countFN+countTN\n",
    "    probAnalisisSama = (countTP+countTN)/totalAnalisis\n",
    "    nilaiAmbiguAhli = (countTP+countFN)/totalAnalisis \n",
    "    nilaiAmbiguSistem = (countTP+countFP)/totalAnalisis\n",
    "    nilaiTidakAmbiguSistem = (countTN+countFN)/totalAnalisis\n",
    "    nilaiTidakAmbiguAhli = (countFP+countTN)/totalAnalisis\n",
    "    probAmbigu = nilaiAmbiguAhli*nilaiAmbiguSistem\n",
    "    probTidakAmbigu = nilaiTidakAmbiguAhli*nilaiTidakAmbiguSistem\n",
    "    probAnalisisBeda = probAmbigu+probTidakAmbigu\n",
    "    nilaiKappa = (probAnalisisSama-probAnalisisBeda)/(1-probAnalisisBeda)\n",
    "    return nilaiKappa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Sistem - Ahli  Ambigu  Tidak Ambigu\n",
      "0        Ambigu       6             0\n",
      "1  Tidak Ambigu       2           108\n",
      "----------------\n",
      "Nilai Kappa:  0.8481675392670154\n"
     ]
    }
   ],
   "source": [
    "countTPVague,countFPVague,countFNVague,countTNVague=hitungConfusionMatrix('vague')\n",
    "nilaiKappaVague = hitungNilaiKappa(countTPVague,countFPVague,countFNVague,countTNVague)\n",
    "dfVague = dataraw(countTPVague,countFPVague,countFNVague,countTNVague)\n",
    "print(dfVague)\n",
    "print('----------------')\n",
    "print('Nilai Kappa: ',nilaiKappaVague)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Sistem - Ahli  Ambigu  Tidak Ambigu\n",
      "0        Ambigu      12             2\n",
      "1  Tidak Ambigu       0           108\n",
      "----------------\n",
      "Nilai Kappa:  0.9139633286318758\n"
     ]
    }
   ],
   "source": [
    "countTPAnalytical,countFPAnalytical,countFNAnalytical,countTNAnalytical=hitungConfusionMatrix('analytical')\n",
    "nilaiKappaAnalytical = hitungNilaiKappa(countTPAnalytical,countFPAnalytical,countFNAnalytical,countTNAnalytical)\n",
    "dfAnalytical = dataraw(countTPAnalytical,countFPAnalytical,countFNAnalytical,countTNAnalytical)\n",
    "print(dfAnalytical)\n",
    "print('----------------')\n",
    "print('Nilai Kappa: ',nilaiKappaAnalytical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Sistem - Ahli  Ambigu  Tidak Ambigu\n",
      "0        Ambigu       7             1\n",
      "1  Tidak Ambigu       2           108\n",
      "----------------\n",
      "Nilai Kappa:  0.809881847475832\n"
     ]
    }
   ],
   "source": [
    "countTPCoordination,countFPCoordination,countFNCoordination,countTNCoordination=hitungConfusionMatrix('coordination')\n",
    "nilaiKappaCoordination = hitungNilaiKappa(countTPCoordination,countFPCoordination,countFNCoordination,countTNCoordination)\n",
    "dfCoordination = dataraw(countTPCoordination,countFPCoordination,countFNCoordination,countTNCoordination)\n",
    "print(dfCoordination)\n",
    "print('----------------')\n",
    "print('Nilai Kappa: ',nilaiKappaCoordination)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Sistem - Ahli  Ambigu  Tidak Ambigu\n",
      "0        Ambigu       0             1\n",
      "1  Tidak Ambigu       4           108\n",
      "----------------\n",
      "Nilai Kappa:  -0.014362657091559714\n"
     ]
    }
   ],
   "source": [
    "countTPPPattachment,countFPPPattachment,countFNPPattachment,countTNPPattachment=hitungConfusionMatrix('pp attachment')\n",
    "nilaiKappaPPattachment = hitungNilaiKappa(countTPPPattachment,countFPPPattachment,countFNPPattachment,countTNPPattachment)\n",
    "dfPPattachment= dataraw(countTPPPattachment,countFPPPattachment,countFNPPattachment,countTNPPattachment)\n",
    "print(dfPPattachment)\n",
    "print('----------------')\n",
    "print('Nilai Kappa: ',nilaiKappaPPattachment)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}