Philipp/reversi
1
1 Fork 0
Code Issues Pull Requests Packages Releases Activity

Repaired a bug in the q learning reword function

Browse Source
This commit is contained in:
Philipp committed 2023-02-22 21:55:08 +01:00
1 parent 138d9df2b9
commit ef122fb6d0
1 file changed
+85 -69
Download Patch File Download Diff File Expand all files Collapse all files
main.ipynb
+85 -69
View File
@@ -954,7 +954,7 @@
" raise InvalidTurn(\"An action should be taken. A turn is possible.\")\n",
" return\n",
"\n",
" # noinspection PyTypeChecker\n",
" # noinspection PyTypeChecker \n",
" if _board[tuple(move.tolist())] != 0:\n",
" raise InvalidTurn(\"This turn is not possible.\")\n",
"\n",
@@ -1961,29 +1961,28 @@
},
{
"cell_type": "code",
"execution_count": 44,
"execution_count": 98,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 0.09677184, 0.0037773 , 0.12190913, 0.03519891, 0.16118614,\n",
" 0.00617017, 0.12490022, -0.03918723, 0.14632847, -0.01240192,\n",
" 0.1016851 , 0.00991888, 0.1295861 , -0.03332988, 0.07552515,\n",
" -0.10090606, 0.14730492, -0.08930635, 0.08367957, -0.09071304,\n",
" 0.1600462 , 0.08287025, 0.22077531, -0.07559336, 0.1789458 ,\n",
" 0.02836975, 0.23077469, 0.01503086, 0.13597608, -0.18159241,\n",
" -0.03167801, -0.23491001, 0.05792499, -0.04478127, 0.06121092,\n",
" -0.04067385, 0.37884519, 0.04386898, 0.17202373, -0.05840784,\n",
" 0.0441777 , -0.14009038, 0.02019953, -0.09193809, 0.15851489,\n",
" 0.08095611, 0.45275764, 0.13625955, 0.36563693, -0.05076633,\n",
" 0.28810459, -0.22580677, -0.16507096, -0.5579012 , -0.033314 ,\n",
" -0.15883 , 0.23115 , -0.45325 , -0.37125 , -0.58125 ,\n",
" -0.21875 , -0.21875 , -0.21875 , -0.21875 , -0.21875 ,\n",
" -0.21875 , -0.21875 , -0.21875 , -0.21875 , -0.21875 ])"
"array([[ 2.26795474e-10, -4.53590947e-11, -2.26795474e-11, ...,\n",
" 0.00000000e+00, 6.80386421e-11, -4.53590947e-11],\n",
" [ 3.23993534e-10, -6.47987067e-11, -3.23993534e-11, ...,\n",
" 0.00000000e+00, 9.71980601e-11, -6.47987067e-11],\n",
" [ 4.62847905e-10, -9.25695810e-11, -4.62847905e-11, ...,\n",
" 0.00000000e+00, 1.38854372e-10, -9.25695810e-11],\n",
" ...,\n",
" [ 4.46428571e-01, -8.92857143e-02, -4.46428571e-02, ...,\n",
" 0.00000000e+00, 1.33928571e-01, -8.92857143e-02],\n",
" [ 4.46428571e-01, -8.92857143e-02, -4.46428571e-02, ...,\n",
" 0.00000000e+00, 1.33928571e-01, -8.92857143e-02],\n",
" [ 4.46428571e-01, -8.92857143e-02, -4.46428571e-02, ...,\n",
" 0.00000000e+00, 1.33928571e-01, -8.92857143e-02]])"
]
},
"execution_count": 44,
"execution_count": 98,
"metadata": {},
"output_type": "execute_result"
}
@@ -2002,10 +2001,10 @@
" gama_table = get_gamma_table(board_history, gamma)\n",
" combined_score = np.zeros_like(gama_table)\n",
" combined_score += calculate_direct_score(board_history) * (\n",
" 1 - who_won_fraction + final_score_fraction\n",
" 1 - (who_won_fraction + final_score_fraction)\n",
" )\n",
" combined_score[-1] += (\n",
" calculate_final_evaluation_for_history(board_history) * final_score_fraction\n",
" calculate_final_evaluation_for_history(board_history) * final_score_fraction / 0.7\n",
" )\n",
" combined_score[-1] += calculate_who_won(board_history) * who_won_fraction\n",
" for turn in range(SIMULATE_TURNS - 1, 0, -1):\n",
@@ -2016,39 +2015,39 @@
"\n",
"\n",
"calculate_q_reword(\n",
" _board_history, gamma=0.8, who_won_fraction=0, final_score_fraction=1\n",
")[:, 0]"
" _board_history, gamma=0.7, who_won_fraction=0, final_score_fraction=1\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 45,
"execution_count": 99,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([-1.53249554e-06, -1.91561943e-06, -2.39452428e-06, -2.99315535e-06,\n",
" -3.74144419e-06, -4.67680524e-06, -5.84600655e-06, -7.30750819e-06,\n",
" -9.13438523e-06, -1.14179815e-05, -1.42724769e-05, -1.78405962e-05,\n",
" -2.23007452e-05, -2.78759315e-05, -3.48449144e-05, -4.35561430e-05,\n",
" -5.44451787e-05, -6.80564734e-05, -8.50705917e-05, -1.06338240e-04,\n",
" -1.32922800e-04, -1.66153499e-04, -2.07691874e-04, -2.59614843e-04,\n",
" -3.24518554e-04, -4.05648192e-04, -5.07060240e-04, -6.33825300e-04,\n",
" -7.92281625e-04, -9.90352031e-04, -1.23794004e-03, -1.54742505e-03,\n",
" -1.93428131e-03, -2.41785164e-03, -3.02231455e-03, -3.77789319e-03,\n",
" -4.72236648e-03, -5.90295810e-03, -7.37869763e-03, -9.22337204e-03,\n",
" -1.15292150e-02, -1.44115188e-02, -1.80143985e-02, -2.25179981e-02,\n",
" -2.81474977e-02, -3.51843721e-02, -4.39804651e-02, -5.49755814e-02,\n",
" -6.87194767e-02, -8.58993459e-02, -1.07374182e-01, -1.34217728e-01,\n",
" -1.67772160e-01, -2.09715200e-01, -2.62144000e-01, -3.27680000e-01,\n",
" -4.09600000e-01, -5.12000000e-01, -6.40000000e-01, -8.00000000e-01,\n",
" -1.00000000e+00, -1.00000000e+00, -1.00000000e+00, -1.00000000e+00,\n",
" -1.00000000e+00, -1.00000000e+00, -1.00000000e+00, -1.00000000e+00,\n",
" -1.00000000e+00, -1.00000000e+00])"
"array([1.53249554e-06, 1.91561943e-06, 2.39452428e-06, 2.99315535e-06,\n",
" 3.74144419e-06, 4.67680524e-06, 5.84600655e-06, 7.30750819e-06,\n",
" 9.13438523e-06, 1.14179815e-05, 1.42724769e-05, 1.78405962e-05,\n",
" 2.23007452e-05, 2.78759315e-05, 3.48449144e-05, 4.35561430e-05,\n",
" 5.44451787e-05, 6.80564734e-05, 8.50705917e-05, 1.06338240e-04,\n",
" 1.32922800e-04, 1.66153499e-04, 2.07691874e-04, 2.59614843e-04,\n",
" 3.24518554e-04, 4.05648192e-04, 5.07060240e-04, 6.33825300e-04,\n",
" 7.92281625e-04, 9.90352031e-04, 1.23794004e-03, 1.54742505e-03,\n",
" 1.93428131e-03, 2.41785164e-03, 3.02231455e-03, 3.77789319e-03,\n",
" 4.72236648e-03, 5.90295810e-03, 7.37869763e-03, 9.22337204e-03,\n",
" 1.15292150e-02, 1.44115188e-02, 1.80143985e-02, 2.25179981e-02,\n",
" 2.81474977e-02, 3.51843721e-02, 4.39804651e-02, 5.49755814e-02,\n",
" 6.87194767e-02, 8.58993459e-02, 1.07374182e-01, 1.34217728e-01,\n",
" 1.67772160e-01, 2.09715200e-01, 2.62144000e-01, 3.27680000e-01,\n",
" 4.09600000e-01, 5.12000000e-01, 6.40000000e-01, 8.00000000e-01,\n",
" 1.00000000e+00, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00,\n",
" 1.00000000e+00, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00,\n",
" 1.00000000e+00, 1.00000000e+00])"
]
},
"execution_count": 45,
"execution_count": 99,
"metadata": {},
"output_type": "execute_result"
}
@@ -2061,33 +2060,33 @@
},
{
"cell_type": "code",
"execution_count": 46,
"execution_count": 100,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 3.09670969, 0.12088712, 3.9011089 , 1.12638612,\n",
" 5.15798265, 0.19747831, 3.99684789, -1.25394014,\n",
" 4.68257483, -0.39678147, 3.25402317, 0.31752896,\n",
" 4.1469112 , -1.066361 , 2.41704875, -3.22868907,\n",
" 4.71413867, -2.85732667, 2.67834167, -2.90207292,\n",
" 5.12240885, 2.65301107, 7.06626383, -2.41717021,\n",
" 5.72853724, 0.91067155, 7.38833944, 0.4854243 ,\n",
" 4.35678037, -5.80402453, -1.00503067, -7.50628834,\n",
" 1.86713958, -1.41607552, 1.9799056 , -1.27511801,\n",
" 12.15610249, 1.44512812, 5.55641015, -1.80448732,\n",
" 1.49439085, -4.38201144, 0.77248571, -2.78439287,\n",
" 5.26950892, 2.83688614, 14.79610768, 4.7451346 ,\n",
" 12.18141825, -1.02322719, 9.97096602, -6.28629248,\n",
" -4.1078656 , -16.384832 , 0.76896 , -2.7888 ,\n",
" 10.264 , -10.92 , -7.4 , -13. ,\n",
" 0. , 0. , 0. , 0. ,\n",
" 0. , 0. , 0. , 0. ,\n",
" 0. , 0. ])"
"array([ 1.80163817e+00, -1.49795229e+00, 1.87755963e+00, -1.40305046e+00,\n",
" 1.99618693e+00, -1.25476634e+00, 4.68154208e+00, -3.98072406e-01,\n",
" 3.25240949e+00, -2.18448813e+00, 3.51938983e+00, -4.35076271e+00,\n",
" 8.11546612e-01, -2.73556673e+00, 2.83054158e+00, -2.11823023e-01,\n",
" 3.48522122e+00, -1.89347347e+00, 1.38315816e+00, -2.02105230e+00,\n",
" 3.72368462e+00, 9.04605778e-01, 4.88075722e+00, -2.64905347e+00,\n",
" 4.38683161e-01, -3.20164605e+00, -2.52057562e-01, -9.06507195e+00,\n",
" -8.13399398e-02, -3.85167492e+00, 6.43540634e+00, 1.79425793e+00,\n",
" 1.09928224e+01, -5.00897198e+00, -1.12149805e-02, -6.26401873e+00,\n",
" 3.41997659e+00, -9.47502926e+00, -8.09378657e+00, -1.38672332e+01,\n",
" -6.08404152e+00, -1.13550519e+01, -2.94381488e+00, -7.42976860e+00,\n",
" 1.96278926e+00, -1.12965134e+01, 2.12935821e+00, -6.08830224e+00,\n",
" -1.36037779e+00, -1.04504722e+01, 3.18690970e+00, -7.26636288e+00,\n",
" 4.66704640e+00, -1.04161920e+01, -6.77024000e+00, -1.22128000e+01,\n",
" 9.84000000e-01, -2.52000000e+00, 6.00000000e-01, -3.00000000e+00,\n",
" 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
" 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
" 0.00000000e+00, 0.00000000e+00])"
]
},
"execution_count": 46,
"execution_count": 100,
"metadata": {},
"output_type": "execute_result"
}
@@ -2813,20 +2812,16 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 86,
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1cc65e04021b411da613d29ff2eda0c8",
"version_major": 2,
"version_minor": 0
},
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAABK1klEQVR4nO3dd1gU1/s28HtBXBZpAiIgINhREYVgjSWxgAUxGk3UqMSegNi/lkRFjaKJJTFRkqgBk9gTsZdYABs2FBuIiCgWiLGBgCCy5/3D1/m5ARQMsMDcn+vaS/bMmZlnGGBuz5RVCCEEiIiIiGRER9sFEBEREZU2BiAiIiKSHQYgIiIikh0GICIiIpIdBiAiIiKSHQYgIiIikh0GICIiIpIdBiAiIiKSHQYgIiIikh0GICIqsxwcHODj41Oovh06dECHDh1KtB4iqjgYgIhI644fP46AgAA8fvz4tf1iYmIQEBCAGzdulEpdRFRxMQARkdYdP34cs2fPzhOA4uLisHLlSul9TEwMZs+ezQBERP9ZJW0XQERUEKVSqe0SiKiC4ggQEWlVQEAAJk+eDABwdHSEQqGAQqHAjRs3NK4BCgkJQd++fQEA7733ntQvPDy8wGVnZ2dj1qxZqFOnDpRKJezs7PC///0P2dnZJb1ZRFTGcQSIiLSqd+/euHr1KtavX4+lS5fCwsICAFCtWjWNfu3atYO/vz+WLVuG6dOnw8nJCQCkf/9NrVajZ8+eOHr0KEaOHAknJydcvHgRS5cuxdWrV7F169YS3S4iKtsYgIhIq5o0aQJXV1esX78evXr1goODQ779atWqhbZt22LZsmXo3LnzG+/4WrduHQ4cOICIiAi8++67Unvjxo0xevRoHD9+HK1bty7GLSGi8oSnwIioQtq8eTOcnJzQoEED3L9/X3q9//77AICwsDAtV0hE2sQRICKqkOLj4xEbG5vnVNpL9+7dK+WKiKgsYQAiogpJrVbD2dkZS5YsyXe6nZ1dKVdERGUJAxARaZ1CoSjWfgBQu3ZtnD9/Hh07dizSfEQkD7wGiIi0rkqVKgDwxidBF7YfAPTr1w937tzReJDiS0+fPkVGRkaR6ySiioMjQESkdW5ubgCAL774Ah9//DH09PTg5eWVp1/Tpk2hq6uLhQsXIjU1FUqlEu+//z4sLS3z9B00aBA2bdqE0aNHIywsDG3atEFubi6uXLmCTZs2Yd++fXjnnXdKfNuIqGxiACIirXN3d8fcuXPx448/Yu/evVCr1UhMTMzTz8rKCj/++CMCAwMxbNgw5ObmIiwsLN8ApKOjg61bt2Lp0qX49ddfERoaCgMDA9SqVQtjx45FvXr1SmPTiKiMUgghhLaLICIiIipNvAaIiIiIZIcBiIiIiGSHAYiIiIhkhwGIiIiIZIcBiIiIiGSHAYiIiIhkp8I/B0itVuPu3bswMjLi4/CJiIjKCSEEnjx5AhsbG+joFP94TYUPQHfv3uWHHhIREZVTt27dgq2tbbEvt8IHICMjIwAvvoHGxsZaroaIiIgKIy0tDXZ2dtJxvLhV+AD08rSXsbExAxAREVE5U1KXr/AiaCIiIpIdBiAiIiKSHQYgIiIikp0Kfw0QEcmDWq3Gs2fPtF0GERWSnp4edHV1tbZ+BiAiKveePXuGxMREqNVqbZdCREVgamoKKysrrTynjwGIiMo1IQSSk5Ohq6sLOzu7EnlgGhEVLyEEMjMzce/ePQCAtbV1qdfAAERE5drz58+RmZkJGxsbGBgYaLscIioklUoFALh37x4sLS1L/XQY/6tEROVabm4uAKBy5cparoSIiurlf1pycnJKfd0MQERUIfCz/ojKH23+3mo1AAUGBsLd3R1GRkawtLREr169EBcXp9EnJSUFgwYNgpWVFapUqQJXV1f8+eefWqqYiIiIKgKtBqCIiAj4+vrixIkT2L9/P3JyctClSxdkZGRIfQYPHoy4uDhs374dFy9eRO/evdGvXz+cO3dOi5UTERFReabVALR37174+PigUaNGcHFxQUhICJKSkhAVFSX1OX78OMaMGYPmzZujVq1a+PLLL2FqaqrRh4iIioePjw8UCgUWLFig0b5161atn2a8ceMGFAoFLC0t8eTJE41pTZs2RUBAgHYKo3KpTF0DlJqaCgAwMzOT2lq3bo2NGzfi4cOHUKvV2LBhA7KystChQ4d8l5GdnY20tDSNFxERFZ6+vj4WLlyIR48eabuUfD158gSLFi3SdhlUzpWZAKRWqzFu3Di0adMGjRs3lto3bdqEnJwcmJubQ6lUYtSoUQgNDUWdOnXyXU5gYCBMTEykl52dXWltAhFRhdCpUydYWVkhMDCwwD5//vknGjVqBKVSCQcHByxevFhjuoODA+bPn4+hQ4fCyMgI9vb2+PnnnzX63Lp1C/369YOpqSnMzMzg7e2NGzduvLG+MWPGYMmSJdIzZPLz6NEjDB48GFWrVoWBgQG6du2K+Ph4aXpISAhMTU2xb98+ODk5wdDQEJ6enkhOTtZYzqpVq+Dk5AR9fX00aNAAK1aseGN9VD6UmQDk6+uLS5cuYcOGDRrtM2bMwOPHj3HgwAGcOXMGEyZMQL9+/XDx4sV8lzNt2jSkpqZKr1u3bpVG+UREFYauri7mz5+P77//Hrdv384zPSoqCv369cPHH3+MixcvIiAgADNmzEBISIhGv8WLF+Odd97BuXPn8Pnnn+Ozzz6TbnTJycmBh4cHjIyMcOTIERw7dkwKIW/6SJP+/fujTp06mDNnToF9fHx8cObMGWzfvh2RkZEQQqBbt24at1tnZmZi0aJF+O2333D48GEkJSVh0qRJ0vS1a9di5syZmDdvHmJjYzF//nzMmDEDa9asKcy3kco6UQb4+voKW1tbcf36dY32a9euCQDi0qVLGu0dO3YUo0aNKtSyU1NTBQCRmppabPUSUdnx9OlTERMTI54+fartUiqEIUOGCG9vbyGEEC1bthRDhw4VQggRGhoqXh4yBgwYIDp37qwx3+TJk0XDhg2l9zVr1hSffPKJ9F6tVgtLS0sRFBQkhBDit99+E/Xr1xdqtVrqk52dLVQqldi3b1++tSUmJgoA4ty5c2Lv3r1CT09PXLt2TQghhIuLi5g1a5YQQoirV68KAOLYsWPSvPfv3xcqlUps2rRJCCFEcHCwACDNL4QQy5cvF9WrV5fe165dW6xbt06jhrlz54pWrVoV9O2jInrd729JH7+1OgIkhICfnx9CQ0Nx6NAhODo6akzPzMwEgDyPttfV1eVn/hARlbCFCxdizZo1iI2N1WiPjY1FmzZtNNratGmD+Ph46cGUANCkSRPpa4VCASsrK+m01fnz53Ht2jUYGRnB0NAQhoaGMDMzQ1ZWFhISEt5Ym4eHB959913MmDEjz7TY2FhUqlQJLVq0kNrMzc1Rv359jW0xMDBA7dq1pffW1tZSfRkZGUhISMCwYcOk+gwNDfHVV18Vqj4q+7T6URi+vr5Yt24dtm3bBiMjI6SkpAAATExMoFKp0KBBA9SpUwejRo3CokWLYG5ujq1bt2L//v3YuXOnNksnIqrw2rVrBw8PD0ybNg0+Pj5Fnl9PT0/jvUKhkP7zmp6eDjc3N6xduzbPfNWqVSvU8hcsWIBWrVph8uTJRa6toPqEEFJ9ALBy5UqNIAVAq59gTsVHqwEoKCgIAPLc0RUcHAwfHx/o6elh9+7dmDp1Kry8vJCeno46depgzZo16NatmxYqJiKSlwULFqBp06aoX7++1Obk5IRjx45p9Dt27Bjq1atX6HDg6uqKjRs3wtLSEsbGxm9VW/PmzdG7d29MnTpVo93JyQnPnz/HyZMn0bp1awDAgwcPEBcXh4YNGxZq2dWrV4eNjQ2uX7+OgQMHvlV9VLZpNQC9TNqvU7duXT75mYhIS5ydnTFw4EAsW7ZMaps4cSLc3d0xd+5cfPTRR4iMjMQPP/xQpDukBg4ciG+++Qbe3t6YM2cObG1tcfPmTWzZsgX/+9//YGtrW6jlzJs3D40aNUKlSv93OKtbty68vb0xYsQI/PTTTzAyMsLUqVNRo0YNeHt7F7rG2bNnw9/fHyYmJvD09ER2djbOnDmDR48eYcKECYVeDpVNZeYuMCIiKpvmzJmjcd2lq6srNm3ahA0bNqBx48aYOXMm5syZU6TTZAYGBjh8+DDs7e3Ru3dvODk5YdiwYcjKypJGhMLDw6FQKF57a3y9evUwdOhQZGVlabQHBwfDzc0NPXr0QKtWrSCEwO7du/Oc9nqd4cOHY9WqVQgODoazszPat2+PkJCQPNerUvmkEIUZhinH0tLSYGJigtTU1LceZiWisisrKwuJiYlwdHSEvr6+tsuhYhQcHIz58+cjJiamSMGFyo/X/f6W9PGbI0BERFQm7d69G/Pnz2f4oRKh1WuAiIiICrJ582Ztl0AVGEeAiIiISHYYgIiIiEh2GICIiIhIdhiAiIiISHYYgIiIiEh2GICIiIhIdhiAiIiISHYYgIiItOTBgwewtLR87Uc9lKQbN25AoVAgOjpaK+v/t59//hl2dnbQ0dHBt99+i4CAADRt2rRY1/Hy4zUeP35crMstLffv34elpSVu376t7VLKPT4IkYhIS+bNmwdvb284ODhouxStS0tLg5+fH5YsWYI+ffrAxMQEarUaY8aM0XZpZYqFhQUGDx6MWbNmYfXq1doup1zjCBARkRZkZmZi9erVGDZsWIF9hBB4/vx5KValPUlJScjJyUH37t1hbW0NAwMDGBoawtzcXNullZqcnJxC9fv000+xdu1aPHz4sIQrqtgYgIiItGD37t1QKpVo2bKl1Pby9MyePXvg5uYGpVKJo0ePIiEhAd7e3qhevToMDQ3h7u6OAwcOaCzPwcEB8+fPx9ChQ2FkZAR7e3v8/PPPGn1OnTqFZs2aQV9fH++88w7OnTuXp66IiAg0b94cSqUS1tbWmDp1qkYI69ChA8aMGYNx48aLine truncated
"text/plain": [
" 0%| | 0/200 [00:00<?, ?it/s]"
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
@@ -2835,7 +2830,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2e88df0594314308b045b579431a574b",
"model_id": "616f06c422444f12bdc42677b1ad8ef4",
"version_major": 2,
"version_minor": 0
},
@@ -2845,6 +2840,27 @@
},
"metadata": {},
"output_type": "display_data"
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"Cell \u001b[1;32mIn[86], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mql_policy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m200\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1000\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[43mRandomPolicy\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mGreedyPolicy\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
"Cell \u001b[1;32mIn[55], line 191\u001b[0m, in \u001b[0;36mQLPolicy.train\u001b[1;34m(self, epochs, batches, batch_size, eval_batch_size, compare_with, save_every_epoch, live_plot)\u001b[0m\n\u001b[0;32m 189\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28mrange\u001b[39m(epochs)):\n\u001b[0;32m 190\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28mrange\u001b[39m(batches)):\n\u001b[1;32m--> 191\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_batch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 192\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining_results\u001b[38;5;241m.\u001b[39mappend(\n\u001b[0;32m 193\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mevaluate_model(compare_with, eval_batch_size)\n\u001b[0;32m 194\u001b[0m )\n\u001b[0;32m 195\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m save_every_epoch:\n",
"Cell \u001b[1;32mIn[55], line 106\u001b[0m, in \u001b[0;36mQLPolicy.train_batch\u001b[1;34m(self, nr_of_games)\u001b[0m\n\u001b[0;32m 105\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mtrain_batch\u001b[39m(\u001b[38;5;28mself\u001b[39m, nr_of_games: \u001b[38;5;28mint\u001b[39m):\n\u001b[1;32m--> 106\u001b[0m x_train, y_train \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate_trainings_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnr_of_games\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 107\u001b[0m y_pred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mneural_network\u001b[38;5;241m.\u001b[39mforward(x_train)\n\u001b[0;32m 108\u001b[0m loss_score \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mloss(y_pred, y_train)\n",
"Cell \u001b[1;32mIn[55], line 71\u001b[0m, in \u001b[0;36mQLPolicy.generate_trainings_data\u001b[1;34m(self, generate_data_size)\u001b[0m\n\u001b[0;32m 68\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgenerate_trainings_data\u001b[39m(\n\u001b[0;32m 69\u001b[0m \u001b[38;5;28mself\u001b[39m, generate_data_size: \u001b[38;5;28mint\u001b[39m\n\u001b[0;32m 70\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mtuple\u001b[39m[torch\u001b[38;5;241m.\u001b[39mTensor, torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[1;32m---> 71\u001b[0m train_boards, train_actions \u001b[38;5;241m=\u001b[39m \u001b[43msimulate_game\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgenerate_data_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 72\u001b[0m action_possible \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m~\u001b[39mnp\u001b[38;5;241m.\u001b[39mall(train_actions[:, :] \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n\u001b[0;32m 73\u001b[0m q_leaning_formatted_action \u001b[38;5;241m=\u001b[39m build_symetry_action(train_boards, train_actions)\n",
"Cell \u001b[1;32mIn[23], line 25\u001b[0m, in \u001b[0;36msimulate_game\u001b[1;34m(nr_of_games, policies, tqdm_on)\u001b[0m\n\u001b[0;32m 23\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m policy_index \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m 24\u001b[0m current_boards \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m\n\u001b[1;32m---> 25\u001b[0m current_boards, action_taken \u001b[38;5;241m=\u001b[39m \u001b[43msingle_turn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcurrent_boards\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpolicy\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 26\u001b[0m action_history_stack[turn_index, :] \u001b[38;5;241m=\u001b[39m action_taken\n\u001b[0;32m 28\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m policy_index \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
"Cell \u001b[1;32mIn[22], line 15\u001b[0m, in \u001b[0;36msingle_turn\u001b[1;34m(current_boards, policy)\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msingle_turn\u001b[39m(\n\u001b[0;32m 2\u001b[0m current_boards: np, policy: GamePolicy\n\u001b[0;32m 3\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mtuple\u001b[39m[np\u001b[38;5;241m.\u001b[39mndarray, np\u001b[38;5;241m.\u001b[39mndarray]:\n\u001b[0;32m 4\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Execute a single turn on a board.\u001b[39;00m\n\u001b[0;32m 5\u001b[0m \n\u001b[0;32m 6\u001b[0m \u001b[38;5;124;03m Places a new stone on the board. Turns captured enemy stones.\u001b[39;00m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 13\u001b[0m \u001b[38;5;124;03m The new game board and the policy vector containing the index of the action used.\u001b[39;00m\n\u001b[0;32m 14\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m---> 15\u001b[0m policy_results \u001b[38;5;241m=\u001b[39m \u001b[43mpolicy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_policy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcurrent_boards\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 17\u001b[0m \u001b[38;5;66;03m# if the constant VERIFY_POLICY is set to true the policy is verified. Should be good though.\u001b[39;00m\n\u001b[0;32m 18\u001b[0m \u001b[38;5;66;03m# todo deactivate the policy verification after some testing.\u001b[39;00m\n\u001b[0;32m 19\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m VERIFY_POLICY:\n",
"Cell \u001b[1;32mIn[19], line 64\u001b[0m, in \u001b[0;36mGamePolicy.get_policy\u001b[1;34m(self, boards)\u001b[0m\n\u001b[0;32m 59\u001b[0m policies[random_choices] \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mrandom\u001b[38;5;241m.\u001b[39mrand(np\u001b[38;5;241m.\u001b[39msum(random_choices), \u001b[38;5;241m8\u001b[39m ,\u001b[38;5;241m8\u001b[39m)\n\u001b[0;32m 61\u001b[0m \u001b[38;5;66;03m# todo talk to team about backpropagation of score and epsilon for greedy factor\u001b[39;00m\n\u001b[0;32m 62\u001b[0m \n\u001b[0;32m 63\u001b[0m \u001b[38;5;66;03m# todo possibly change this function to only validate the purpose turn and not all turns\u001b[39;00m\n\u001b[1;32m---> 64\u001b[0m possible_turns \u001b[38;5;241m=\u001b[39m \u001b[43mget_possible_turns\u001b[49m\u001b[43m(\u001b[49m\u001b[43mboards\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 65\u001b[0m policies[possible_turns \u001b[38;5;241m==\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1.0\u001b[39m\n\u001b[0;32m 66\u001b[0m max_indices \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m 67\u001b[0m np\u001b[38;5;241m.\u001b[39munravel_index(policy\u001b[38;5;241m.\u001b[39margmax(), policy\u001b[38;5;241m.\u001b[39mshape) \u001b[38;5;28;01mfor\u001b[39;00m policy \u001b[38;5;129;01min\u001b[39;00m policies\n\u001b[0;32m 68\u001b[0m ]\n",
"Cell \u001b[1;32mIn[13], line 60\u001b[0m, in \u001b[0;36mget_possible_turns\u001b[1;34m(boards, tqdm_on)\u001b[0m\n\u001b[0;32m 58\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m poss_turns[game, idx, idy]:\n\u001b[0;32m 59\u001b[0m position \u001b[38;5;241m=\u001b[39m idx, idy\n\u001b[1;32m---> 60\u001b[0m poss_turns[game, idx, idy] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43many\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[0;32m 61\u001b[0m \u001b[43m \u001b[49m\u001b[43m_recursive_steps\u001b[49m\u001b[43m(\u001b[49m\u001b[43mboards\u001b[49m\u001b[43m[\u001b[49m\u001b[43mgame\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdirection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mposition\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m>\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\n\u001b[0;32m 62\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mdirection\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mDIRECTIONS\u001b[49m\n\u001b[0;32m 63\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m poss_turns\n",
"Cell \u001b[1;32mIn[13], line 61\u001b[0m, in \u001b[0;36m<genexpr>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m 58\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m poss_turns[game, idx, idy]:\n\u001b[0;32m 59\u001b[0m position \u001b[38;5;241m=\u001b[39m idx, idy\n\u001b[0;32m 60\u001b[0m poss_turns[game, idx, idy] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28many\u001b[39m(\n\u001b[1;32m---> 61\u001b[0m \u001b[43m_recursive_steps\u001b[49m\u001b[43m(\u001b[49m\u001b[43mboards\u001b[49m\u001b[43m[\u001b[49m\u001b[43mgame\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdirection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mposition\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m 62\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m direction \u001b[38;5;129;01min\u001b[39;00m DIRECTIONS\n\u001b[0;32m 63\u001b[0m )\n\u001b[0;32m 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m poss_turns\n",
"Cell \u001b[1;32mIn[13], line 19\u001b[0m, in \u001b[0;36m_recursive_steps\u001b[1;34m(board, rec_direction, rec_position, step_one)\u001b[0m\n\u001b[0;32m 7\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Check if a player can place a stone on the board specified in the direction specified and direction specified.\u001b[39;00m\n\u001b[0;32m 8\u001b[0m \n\u001b[0;32m 9\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 16\u001b[0m \u001b[38;5;124;03m True if a turn is possible for possition and direction on the board defined.\u001b[39;00m\n\u001b[0;32m 17\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 18\u001b[0m rec_position \u001b[38;5;241m=\u001b[39m rec_position \u001b[38;5;241m+\u001b[39m rec_direction\n\u001b[1;32m---> 19\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43many\u001b[49m\u001b[43m(\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrec_position\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m>\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mBOARD_SIZE\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m|\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mrec_position\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m<\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[0;32m 20\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m 21\u001b[0m next_field \u001b[38;5;241m=\u001b[39m board[\u001b[38;5;28mtuple\u001b[39m(rec_position\u001b[38;5;241m.\u001b[39mtolist())]\n",
"File \u001b[1;32m<__array_function__ internals>:180\u001b[0m, in \u001b[0;36many\u001b[1;34m(*args, **kwargs)\u001b[0m\n",
"\u001b[1;31mKeyboardInterrupt\u001b[0m: "
]
}
],
"source": [