notebooks/Process_AUT_GT.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/massivetexts/llm_aut_study/blob/main/notebooks/Process_AUT_GT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eXqu-c24fsZs"
      },
      "source": [
        "This notebook collects, normalizes and processes AUT human judged ground truth from various sources. This is the data used in [Beyond Semantic Distance: Automated Scoring of Divergent Thinking Greatly Improves with Large Language Models](https://www.researchgate.net/publication/363456838_Beyond_Semantic_Distance_Automated_Scoring_of_Divergent_Thinking_Greatly_Improves_with_Large_Language_Models/stats).\n",
        "\n",
        "This is an assortment of studies, with different demographics, goals, and test setups. It is most appropriate for supervised learning of automated scoring, where we're not necessarily trying to learn about the participants but the raters in general."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sAfU9ApyfppM",
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c3202312-63b7-4ccd-ecf9-7404c6f620bb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: xlrd in /usr/local/lib/python3.10/dist-packages (2.0.1)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m198.6/198.6 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for littleutils (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "import dataclasses\n",
        "#@title import and setup defs\n",
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "import hashlib\n",
        "from pathlib import Path\n",
        "from sklearn.model_selection import train_test_split\n",
        "import json\n",
        "\n",
        "!pip install --upgrade xlrd\n",
        "!pip -qq install pingouin\n",
        "import pingouin as pg\n",
        "\n",
        "def simple_stats(df, rater_cols=False):\n",
        "    print(\"# of prompts\", len(df.prompt.unique()))\n",
        "    print(\"# of participants\", len(df.participant.unique()))\n",
        "    print(\"# of data points\", len(df))\n",
        "    print(\"Prompts\", df.prompt.unique())\n",
        "    if rater_cols:\n",
        "        print(\"# of raters\", len(rater_cols))\n",
        "        print(\"Intraclass correlation coefficients (report ICC2k)\")\n",
        "        x = df[['id']+rater_cols].melt(id_vars='id', var_name='rater', value_name='rating')\n",
        "        icc = pg.intraclass_corr(data=x, targets='id', raters='rater', ratings='rating', nan_policy='omit')\n",
        "        display(icc.round(2))\n",
        "\n",
        "include_rater_std = False #@param {type:'boolean'}\n",
        "returncols = ['type', 'src', 'question', 'prompt', 'response', 'id', 'target', 'participant', 'response_num']\n",
        "if include_rater_std:\n",
        "    returncols += ['rating_std']\n",
        "\n",
        "def prep_general(data, src, rater_cols=None, return_full=False,\n",
        "                 aggregate_scores=True,\n",
        "                 drop_noresponse=True, include_stats=True, round_adjust=False,\n",
        "                 include_rater_std=False, overwrite_q=False, inputrange=None):\n",
        "    ''' General cleaning that repeats for multiple datasets.\n",
        "\n",
        "    If aggregate_scores is False, this will (eventually) return original individual rater scores, rather than multi-rater.\n",
        "        I haven't implemented this disagregation yet because I'm not sure how well disagregated\n",
        "        scored could be used in a proper training, without data leakage.\n",
        "    '''\n",
        "    if not rater_cols:\n",
        "        # assume columns that say 'rater'\n",
        "        rater_cols = [col for col in data.columns if 'rater' in col.lower()]\n",
        "\n",
        "    print(\"Rater cols:\", rater_cols)\n",
        "\n",
        "    if drop_noresponse:\n",
        "        data = data[~data.response.isna()]\n",
        "\n",
        "    data['src'] = src\n",
        "    data['avg_rating'] = data[rater_cols].mean(1)\n",
        "    if round_adjust:\n",
        "        # add a tiny bit to avg rater to round in direction of median. Only need if there are tiebreakers\n",
        "        data['median_rating'] = data[rater_cols].median(1)\n",
        "        data['avg_rating'] = data.avg_rating + (data.median_rating - data.avg_rating).div(10**3)\n",
        "    if include_rater_std:\n",
        "        data['rating_std'] = data[rater_cols].std(1)\n",
        "\n",
        "    data['target'] = normalize_values(data.avg_rating, oldrange=inputrange)\n",
        "\n",
        "    missing = data.target.isna()\n",
        "    if missing.sum():\n",
        "        print(f'Dropping {missing.sum()} unrated items')\n",
        "        data = data[~missing]\n",
        "\n",
        "    data['participant'] = src + data['participant'].astype(str)\n",
        "\n",
        "    if 'type' not in data.columns:\n",
        "        print('assuming type=\"uses\"')\n",
        "        data['type'] = 'uses'\n",
        "\n",
        "    if ('question' not in data.columns) or overwrite_q:\n",
        "        try:\n",
        "            assert data.type.unique().tolist() == ['uses']\n",
        "        except:\n",
        "            raise Exception(\"can't infer question format for anything other than uses; please explicitly supply a 'question' column\")\n",
        "        data.loc[data.type=='uses', 'question'] = data.loc[data.type=='uses', 'prompt'].apply(lambda x: f\"What is a surprising use for a {x.upper()}?\")\n",
        "\n",
        "    idhash = (data.participant+data.response).apply(lambda x: hashlib.md5(x.encode('utf-8')).hexdigest()[:6])\n",
        "    # task can be a custom string that is used instead of prompt (e.g. 'g2_red' instead of 'red')\n",
        "    data['id'] = f'{src}_' + data['task' if 'task' in data.columns else 'prompt'].astype(str) + '-' + idhash\n",
        "\n",
        "    if include_stats:\n",
        "        simple_stats(data, rater_cols)\n",
        "\n",
        "    if return_full:\n",
        "        return data\n",
        "    else:\n",
        "        cols = ['type', 'src', 'question', 'prompt', 'response', 'id', 'target', 'participant', 'response_num']\n",
        "        if include_rater_std:\n",
        "            cols += ['rating_std']\n",
        "        return data[cols]\n",
        "\n",
        "datasets = dict()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "p6OHgu0tat-Q"
      },
      "outputs": [],
      "source": [
        "base_dir = 'drive/MyDrive/Grants/MOTES/Data/aut_ground_truth' #@param { type: 'string' }\n",
        "base_dir = Path(base_dir)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# All AUT ICC2ks, to get a sense of reliability\n",
        "x = np.array([0.85, 0.69, 0.79, 0.48, 0.67, 0.81, 0.67, 0.72])\n",
        "np.mean(x), np.std(x).round(3), np.median(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_7bna3Sx8A39",
        "outputId": "733784fb-01c3-4899-f5d3-52ee4018cb43"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.71, 0.108, 0.705)"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "47HJ3XN7asiY"
      },
      "outputs": [],
      "source": [
        "#@title Normalization Settings\n",
        "#@markdown Ensure that all the datasets have the same scale\n",
        "scale_min = 1 #@param{type:'number'}\n",
        "scale_max = 5 #@param{type:'number'}\n",
        "#@markdown Round to nearest 0.1 by default.\n",
        "round_func = lambda x: x.round(1) #@param [\"lambda x: x.multiply(2).round(0).div(2)\", \"None\", \"lambda x: x.round(1)\"] {type:\"raw\"}\n",
        "\n",
        "def normalize_values(series, outrange=(1,5), oldrange=None,\n",
        "                     round_func=round_func):\n",
        "    min, max = outrange\n",
        "    if oldrange:\n",
        "        oldmin, oldmax = oldrange\n",
        "    else:\n",
        "        oldmin, oldmax = series.min(), series.max()\n",
        "\n",
        "    x = (series - oldmin)/(oldmax-oldmin)\n",
        "    x = min + (max-min)*x\n",
        "    if round_func:\n",
        "        # none of our data has even numbers of raters, so there shouldn't be need for rounding tiebreakers\n",
        "        # but for future proofing, add or subtract a tiny number randomly, so that rounding can go either way\n",
        "        modifier = (2*np.random.randint(0,2,size=x.shape)-1)/10**5\n",
        "        x = round_func(x+modifier)\n",
        "    return x\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zltQcwkXeRvk"
      },
      "source": [
        "## Normalize Dumas et al 2020\n",
        "\n",
        "Download from OSF."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2NxCzyyjeF1S",
        "outputId": "8870286f-a488-4502-db92-950956c7f54f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Rater cols: ['rater1', 'rater2', 'rater3', 'rater4']\n",
            "assuming type=\"uses\"\n",
            "# of prompts 10\n",
            "# of participants 92\n",
            "# of data points 5490\n",
            "Prompts ['book' 'bottle' 'brick' 'fork' 'pants' 'rope' 'shoe' 'shovel' 'table'\n",
            " 'tire']\n",
            "# of raters 4\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F   df1    df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.57  6.27  5376  16131   0.0  [0.56, 0.58]\n",
              "1   ICC2     Single random raters  0.58  7.78  5376  16128   0.0  [0.49, 0.65]\n",
              "2   ICC3      Single fixed raters  0.63  7.78  5376  16128   0.0  [0.62, 0.64]\n",
              "3  ICC1k  Average raters absolute  0.84  6.27  5376  16131   0.0  [0.83, 0.85]\n",
              "4  ICC2k    Average random raters  0.85  7.78  5376  16128   0.0  [0.79, 0.88]\n",
              "5  ICC3k     Average fixed raters  0.87  7.78  5376  16128   0.0  [0.87, 0.88]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-45337110-38d6-4703-a06b-79a822f96d14\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.57</td>\n",
              "      <td>6.27</td>\n",
              "      <td>5376</td>\n",
              "      <td>16131</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.56, 0.58]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.58</td>\n",
              "      <td>7.78</td>\n",
              "      <td>5376</td>\n",
              "      <td>16128</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.49, 0.65]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.63</td>\n",
              "      <td>7.78</td>\n",
              "      <td>5376</td>\n",
              "      <td>16128</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.62, 0.64]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.84</td>\n",
              "      <td>6.27</td>\n",
              "      <td>5376</td>\n",
              "      <td>16131</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.83, 0.85]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.85</td>\n",
              "      <td>7.78</td>\n",
              "      <td>5376</td>\n",
              "      <td>16128</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.79, 0.88]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.87</td>\n",
              "      <td>7.78</td>\n",
              "      <td>5376</td>\n",
              "      <td>16128</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.87, 0.88]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-45337110-38d6-4703-a06b-79a822f96d14')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-45337110-38d6-4703-a06b-79a822f96d14 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-45337110-38d6-4703-a06b-79a822f96d14');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      type    src                               question prompt  \\\n",
              "326   uses  dod20   What is a surprising use for a TIRE?   tire   \n",
              "1571  uses  dod20  What is a surprising use for a BRICK?  brick   \n",
              "\n",
              "                response                  id  target participant  response_num  \n",
              "326                swing   dod20_tire-6d8986     2.3      dod208             3  \n",
              "1571  table leg balancer  dod20_brick-7784ed     2.8     dod2016             4  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1d0cdb61-2d48-48fa-a35b-934196e13d11\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>326</th>\n",
              "      <td>uses</td>\n",
              "      <td>dod20</td>\n",
              "      <td>What is a surprising use for a TIRE?</td>\n",
              "      <td>tire</td>\n",
              "      <td>swing</td>\n",
              "      <td>dod20_tire-6d8986</td>\n",
              "      <td>2.3</td>\n",
              "      <td>dod208</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1571</th>\n",
              "      <td>uses</td>\n",
              "      <td>dod20</td>\n",
              "      <td>What is a surprising use for a BRICK?</td>\n",
              "      <td>brick</td>\n",
              "      <td>table leg balancer</td>\n",
              "      <td>dod20_brick-7784ed</td>\n",
              "      <td>2.8</td>\n",
              "      <td>dod2016</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1d0cdb61-2d48-48fa-a35b-934196e13d11')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1d0cdb61-2d48-48fa-a35b-934196e13d11 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1d0cdb61-2d48-48fa-a35b-934196e13d11');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "!wget -q -O dod20.csv https://osf.io/download/u3yv4/\n",
        "\n",
        "src = 'dod20'\n",
        "paca = pd.read_csv('dod20.csv', index_col=0)\n",
        "paca = paca[~paca.response.str.contains('!!!')]\n",
        "datasets[src] = prep_general(paca, src, include_rater_std=include_rater_std)\n",
        "datasets[src].sample(2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gIwAL3uqaikc"
      },
      "source": [
        "## Process Silvia et 2009\n",
        "\n",
        "Silvia, P. J., Nusbaum, E. C., Berg, C., Martin, C., & O'Connor, A. (2009). Openness to experience, plasticity, and creativity: Exploring lower-order, high-order, and interactive effects. Journal of Research in Personality, 43(6), 1087–1090. https://doi.org/10.1016/j.jrp.2009.04.015"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pM8plqKLe-Iv",
        "outputId": "f04ba20b-f0a0-499d-db20-3d3989d3185a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Rater cols: ['rater_1', 'rater_2', 'rater_3', 'rater_4']\n",
            "Dropping 10 unrated items\n",
            "assuming type=\"uses\"\n",
            "# of prompts 3\n",
            "# of participants 202\n",
            "# of data points 4099\n",
            "Prompts ['brick' 'knife' 'box']\n",
            "# of raters 4\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F   df1    df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.33  2.97  4097  12294   0.0  [0.31, 0.35]\n",
              "1   ICC2     Single random raters  0.36  4.10  4097  12291   0.0  [0.25, 0.45]\n",
              "2   ICC3      Single fixed raters  0.44  4.10  4097  12291   0.0  [0.42, 0.45]\n",
              "3  ICC1k  Average raters absolute  0.66  2.97  4097  12294   0.0  [0.65, 0.68]\n",
              "4  ICC2k    Average random raters  0.69  4.10  4097  12291   0.0  [0.57, 0.77]\n",
              "5  ICC3k     Average fixed raters  0.76  4.10  4097  12291   0.0  [0.74, 0.77]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6bea418a-4806-4204-a2ea-0956c0c2d24d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.33</td>\n",
              "      <td>2.97</td>\n",
              "      <td>4097</td>\n",
              "      <td>12294</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.31, 0.35]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.36</td>\n",
              "      <td>4.10</td>\n",
              "      <td>4097</td>\n",
              "      <td>12291</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.25, 0.45]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.44</td>\n",
              "      <td>4.10</td>\n",
              "      <td>4097</td>\n",
              "      <td>12291</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.42, 0.45]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.66</td>\n",
              "      <td>2.97</td>\n",
              "      <td>4097</td>\n",
              "      <td>12294</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.65, 0.68]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.69</td>\n",
              "      <td>4.10</td>\n",
              "      <td>4097</td>\n",
              "      <td>12291</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.57, 0.77]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.76</td>\n",
              "      <td>4.10</td>\n",
              "      <td>4097</td>\n",
              "      <td>12291</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.74, 0.77]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6bea418a-4806-4204-a2ea-0956c0c2d24d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6bea418a-4806-4204-a2ea-0956c0c2d24d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6bea418a-4806-4204-a2ea-0956c0c2d24d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "!wget -q -O silvia09.csv https://osf.io/download/qdrv8/\n",
        "src = 'snbmo09'\n",
        "data = pd.read_csv('silvia09.csv').rename(columns={'subject':'participant','response_order':'response_num'})\n",
        "data['prompt'] = data.task.apply(lambda x: x.split('_')[-1])\n",
        "datasets[src] = prep_general(data, src, include_rater_std=include_rater_std)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "datasets[src].sample(1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cTj8vh0wEc4j",
        "outputId": "f9fc87c9-c4d7-4079-c192-da8ce35c52f8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    type      src                               question prompt  response  \\\n",
              "11  uses  snbmo09  What is a surprising use for a KNIFE?  knife  push pin   \n",
              "\n",
              "                        id  target participant  response_num  \n",
              "11  snbmo09_2_knife-c90814     2.3    snbmo091             1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ab920ba7-969b-4bf6-ae8c-1ba2badedb11\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>uses</td>\n",
              "      <td>snbmo09</td>\n",
              "      <td>What is a surprising use for a KNIFE?</td>\n",
              "      <td>knife</td>\n",
              "      <td>push pin</td>\n",
              "      <td>snbmo09_2_knife-c90814</td>\n",
              "      <td>2.3</td>\n",
              "      <td>snbmo091</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ab920ba7-969b-4bf6-ae8c-1ba2badedb11')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ab920ba7-969b-4bf6-ae8c-1ba2badedb11 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ab920ba7-969b-4bf6-ae8c-1ba2badedb11');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Process Hass 2017\n",
        "\n",
        "Hass, R. W. (2017). Semantic search during divergent thinking. Cognition, 166, 344–357. https://doi.org/10.1016/j.cognition.2017.05.039\n",
        "\n",
        "Data at https://osf.io/ng598.\n",
        "\n",
        "This study looked at uses for *bottle* and *brick*. There were 54 participants after data cleaning.\n",
        "\n",
        "Rating was on a 5-point scale. For verification, their reported inter-rater reliability was ICC2k was 0.80 for brick and 0.78 for bottle, which is about what we see below.\n",
        "\n",
        "The rating data was stoplisted, so I need to reconstruct the original responses here."
      ],
      "metadata": {
        "id": "jn8ziOcamzaH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget -q -O hass17ratings.xslx https://osf.io/download/mcykr/ # rater scores\n",
        "!wget -q -O hass17responses1.xslx https://osf.io/download/27bx8/ # responses 1\n",
        "!wget -q -O hass17responses2.xslx https://osf.io/download/rzvyd/ # responses 2"
      ],
      "metadata": {
        "id": "OJaYB4X6mtTn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "src = 'hass17'\n",
        "all_ratings = []\n",
        "for sheet, prompt in [('br_exp1', 'brick'),('br_exp2', 'brick'),('bot_exp1', 'bottle'),('bot_exp2', 'bottle')]:\n",
        "    data = pd.read_excel('hass17ratings.xslx', sheet_name=sheet) #.rename(columns={'subject':'participant','response_order':'response_num'})\n",
        "    data['prompt'] = prompt\n",
        "    all_ratings.append(data)\n",
        "hassratings = pd.concat(all_ratings).rename(columns={'response':'cleaned'})\n",
        "participants = pd.concat([pd.read_excel('hass17responses1.xslx'), pd.read_excel('hass17responses2.xslx')])"
      ],
      "metadata": {
        "id": "c2uGiyokCtkG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# melt original responses to long, reconstructe the cleaned columns, then join with ratings\n",
        "long_part = participants.melt(id_vars='ID', value_name='response').rename(columns={'ID':'participant'})\n",
        "long_part = long_part[long_part.variable.str.contains('resp') & ~long_part.variable.str.contains('time')].dropna()\n",
        "long_part[['prompt', 'response_num']] = long_part.variable.str.split('_', expand=True)\n",
        "long_part.loc[long_part.prompt.str.contains('resp1'), 'prompt'] = 'bottle'\n",
        "long_part.loc[long_part.prompt.str.contains('resp2'), 'prompt'] = 'brick'\n",
        "long_part.sample(10)\n",
        "\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "stops = stopwords.words('english')\n",
        "# not sure which list the study used, so just adjust based on testing\n",
        "stops += ['could']\n",
        "stops = [w for w in stops if w not in ['can']]\n",
        "stops = set(stops)\n",
        "\n",
        "def stop_clean(x):\n",
        "    x = x.lower()\n",
        "    x = x.replace('i.e.', 'e') # quirk of the tokenization in original study\n",
        "    for c in list(\"/\\\\'-()\"):\n",
        "        x = x.replace(c, '')\n",
        "    words = [word for word in word_tokenize(x) if word not in stops]\n",
        "    return \" \".join(words)\n",
        "\n",
        "long_part['cleaned'] = long_part.response.apply(stop_clean)\n",
        "hass07 = long_part.merge(hassratings, how='left', on=['prompt', 'cleaned'])\n",
        "datasets[src] = prep_general(hass07, src, rater_cols=['r1','r2','r3'], include_rater_std=include_rater_std)"
      ],
      "metadata": {
        "id": "FMGwqsjhDHgY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 451
        },
        "outputId": "a7fd7b11-6057-4444-b2c7-b53e00e75e89"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Rater cols: ['r1', 'r2', 'r3']\n",
            "assuming type=\"uses\"\n",
            "# of prompts 2\n",
            "# of participants 57\n",
            "# of data points 1093\n",
            "Prompts ['bottle' 'brick']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F  df1   df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.55  4.68  794  1590   0.0  [0.51, 0.59]\n",
              "1   ICC2     Single random raters  0.56  5.04  794  1588   0.0    [0.5, 0.6]\n",
              "2   ICC3      Single fixed raters  0.57  5.04  794  1588   0.0  [0.54, 0.61]\n",
              "3  ICC1k  Average raters absolute  0.79  4.68  794  1590   0.0  [0.76, 0.81]\n",
              "4  ICC2k    Average random raters  0.79  5.04  794  1588   0.0  [0.75, 0.82]\n",
              "5  ICC3k     Average fixed raters  0.80  5.04  794  1588   0.0  [0.78, 0.82]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3709eb54-9be9-4c17-b94b-57d86339581d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.55</td>\n",
              "      <td>4.68</td>\n",
              "      <td>794</td>\n",
              "      <td>1590</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.51, 0.59]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.56</td>\n",
              "      <td>5.04</td>\n",
              "      <td>794</td>\n",
              "      <td>1588</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.5, 0.6]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.57</td>\n",
              "      <td>5.04</td>\n",
              "      <td>794</td>\n",
              "      <td>1588</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.54, 0.61]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.79</td>\n",
              "      <td>4.68</td>\n",
              "      <td>794</td>\n",
              "      <td>1590</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.76, 0.81]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.79</td>\n",
              "      <td>5.04</td>\n",
              "      <td>794</td>\n",
              "      <td>1588</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.75, 0.82]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.80</td>\n",
              "      <td>5.04</td>\n",
              "      <td>794</td>\n",
              "      <td>1588</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.78, 0.82]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3709eb54-9be9-4c17-b94b-57d86339581d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3709eb54-9be9-4c17-b94b-57d86339581d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3709eb54-9be9-4c17-b94b-57d86339581d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vJqyd2UHDCMZ"
      },
      "source": [
        "## Process Silvia et al Data 2008\n",
        "\n",
        "From https://osf.io/dh7ey, this was the order of creativity tasks:\n",
        "\n",
        "1. Please list all of the creative, unusual uses for a brick that you can think of.\n",
        "2. Please list all of the creative, unusual instances of things that are round that you can think of.\n",
        "3. Imagine that people no longer needed to sleep. Please list creative, unusual consequences that would follow.\n",
        "4. Please list all of the creative, unusual uses for a knife that you can think of.\n",
        "5. Please list all of the creative, unusual instances of things that will make a noise that you can think of.\n",
        "6. Imagine that everyone shrank to 12 inches tall. Please list creative, unusual consequences that would follow.\n",
        "\n",
        "Numbers 1 and 4 are AUT.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7jNMs7mcDh_A",
        "outputId": "8e5e5750-81fa-46ec-dee0-5826d80328c1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/2.7 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.2/2.7 MB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.7/2.7 MB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/2.7 MB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hArchive:  SilvaStudy2.zip\n",
            "  inflating: DT_Responses_PACA_2008_Study_2.sav  \n",
            "  inflating: Summary_Data_Average_T2_and_Snapshot_PACA_2008_Study_2.sav  \n",
            "  inflating: PACA_2008_Input.inp     \n",
            "  inflating: DT.dat                  \n"
          ]
        }
      ],
      "source": [
        "!pip -q install pyreadstat\n",
        "!wget -q -O SilvaStudy2.zip https://files.osf.io/v1/resources/4ketx/providers/osfstorage/5dd70d1f83135e000ec3c242/?zip=\n",
        "!unzip -o SilvaStudy2.zip\n",
        "import pyreadstat"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vhVizY5jg1v6",
        "outputId": "3ffe6030-2ffc-48e0-a6cf-e975594a75e3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Rater cols: ['rater1', 'rater2', 'rater3']\n",
            "Dropping 37 unrated items\n",
            "# of prompts 6\n",
            "# of participants 242\n",
            "# of data points 11490\n",
            "Prompts ['brick' 'round' 'no sleep' 'knife' 'noise' 'shrank']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F    df1    df2  pval  \\\n",
              "0   ICC1   Single raters absolute  0.12  1.40  11488  22978   0.0   \n",
              "1   ICC2     Single random raters  0.20  2.16  11488  22976   0.0   \n",
              "2   ICC3      Single fixed raters  0.28  2.16  11488  22976   0.0   \n",
              "3  ICC1k  Average raters absolute  0.29  1.40  11488  22978   0.0   \n",
              "4  ICC2k    Average random raters  0.43  2.16  11488  22976   0.0   \n",
              "5  ICC3k     Average fixed raters  0.54  2.16  11488  22976   0.0   \n",
              "\n",
              "          CI95%  \n",
              "0  [0.11, 0.13]  \n",
              "1   [0.09, 0.3]  \n",
              "2  [0.27, 0.29]  \n",
              "3  [0.26, 0.31]  \n",
              "4  [0.22, 0.57]  \n",
              "5  [0.52, 0.55]  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-20c62b3c-a9ea-4950-82b1-efc21b333756\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.12</td>\n",
              "      <td>1.40</td>\n",
              "      <td>11488</td>\n",
              "      <td>22978</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.11, 0.13]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.20</td>\n",
              "      <td>2.16</td>\n",
              "      <td>11488</td>\n",
              "      <td>22976</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.09, 0.3]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.28</td>\n",
              "      <td>2.16</td>\n",
              "      <td>11488</td>\n",
              "      <td>22976</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.27, 0.29]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.29</td>\n",
              "      <td>1.40</td>\n",
              "      <td>11488</td>\n",
              "      <td>22978</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.26, 0.31]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.43</td>\n",
              "      <td>2.16</td>\n",
              "      <td>11488</td>\n",
              "      <td>22976</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.22, 0.57]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.54</td>\n",
              "      <td>2.16</td>\n",
              "      <td>11488</td>\n",
              "      <td>22976</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.52, 0.55]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-20c62b3c-a9ea-4950-82b1-efc21b333756')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-20c62b3c-a9ea-4950-82b1-efc21b333756 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-20c62b3c-a9ea-4950-82b1-efc21b333756');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Testing stats just for AUT (burczak et al reported ICC2k as 0.48)\n",
            "Rater cols: ['rater1', 'rater2', 'rater3']\n",
            "# of prompts 2\n",
            "# of participants 241\n",
            "# of data points 3425\n",
            "Prompts ['brick' 'knife']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F   df1   df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.12  1.42  3424  6850   0.0   [0.1, 0.15]\n",
              "1   ICC2     Single random raters  0.23  2.76  3424  6848   0.0  [0.06, 0.39]\n",
              "2   ICC3      Single fixed raters  0.37  2.76  3424  6848   0.0  [0.35, 0.39]\n",
              "3  ICC1k  Average raters absolute  0.30  1.42  3424  6850   0.0  [0.26, 0.34]\n",
              "4  ICC2k    Average random raters  0.48  2.76  3424  6848   0.0  [0.15, 0.65]\n",
              "5  ICC3k     Average fixed raters  0.64  2.76  3424  6848   0.0  [0.62, 0.66]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c9b60f4b-0992-47fc-8a74-742e28fd1581\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.12</td>\n",
              "      <td>1.42</td>\n",
              "      <td>3424</td>\n",
              "      <td>6850</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.1, 0.15]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.23</td>\n",
              "      <td>2.76</td>\n",
              "      <td>3424</td>\n",
              "      <td>6848</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.06, 0.39]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.37</td>\n",
              "      <td>2.76</td>\n",
              "      <td>3424</td>\n",
              "      <td>6848</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.35, 0.39]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.30</td>\n",
              "      <td>1.42</td>\n",
              "      <td>3424</td>\n",
              "      <td>6850</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.26, 0.34]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.48</td>\n",
              "      <td>2.76</td>\n",
              "      <td>3424</td>\n",
              "      <td>6848</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.15, 0.65]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.64</td>\n",
              "      <td>2.76</td>\n",
              "      <td>3424</td>\n",
              "      <td>6848</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.62, 0.66]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c9b60f4b-0992-47fc-8a74-742e28fd1581')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c9b60f4b-0992-47fc-8a74-742e28fd1581 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c9b60f4b-0992-47fc-8a74-742e28fd1581');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "              type      src  \\\n",
              "3775     instances  setal08   \n",
              "9062  consequences  setal08   \n",
              "\n",
              "                                               question    prompt  \\\n",
              "3775          What is a surprising thing that is ROUND?     round   \n",
              "9062  What would be a surprising consequence if PEOP...  no sleep   \n",
              "\n",
              "                     response                  id  target participant  \\\n",
              "3775                  circles  setal08_2.0-1210ef     1.0   setal0884   \n",
              "9062  experience more of life  setal08_3.0-82d3c9     2.3  setal08193   \n",
              "\n",
              "      response_num  \n",
              "3775          12.0  \n",
              "9062           7.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8895ea58-dba2-41c6-a358-ca50145cce8b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3775</th>\n",
              "      <td>instances</td>\n",
              "      <td>setal08</td>\n",
              "      <td>What is a surprising thing that is ROUND?</td>\n",
              "      <td>round</td>\n",
              "      <td>circles</td>\n",
              "      <td>setal08_2.0-1210ef</td>\n",
              "      <td>1.0</td>\n",
              "      <td>setal0884</td>\n",
              "      <td>12.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9062</th>\n",
              "      <td>consequences</td>\n",
              "      <td>setal08</td>\n",
              "      <td>What would be a surprising consequence if PEOP...</td>\n",
              "      <td>no sleep</td>\n",
              "      <td>experience more of life</td>\n",
              "      <td>setal08_3.0-82d3c9</td>\n",
              "      <td>2.3</td>\n",
              "      <td>setal08193</td>\n",
              "      <td>7.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8895ea58-dba2-41c6-a358-ca50145cce8b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8895ea58-dba2-41c6-a358-ca50145cce8b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8895ea58-dba2-41c6-a358-ca50145cce8b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "src = 'setal08'\n",
        "promptref = ['brick', 'round', 'no sleep', 'knife', 'noise', 'shrank']\n",
        "qref = [\"What is a surprising use for a BRICK?\",\n",
        "        \"What is a surprising thing that is ROUND?\",\n",
        "        \"What would be a surprising consequence if PEOPLE NEEDED NO SLEEP?\",\n",
        "        \"What is a surprising use for a KNIFE?\",\n",
        "        \"What is a surprising thing that makes a NOISE?\",\n",
        "        \"What would be a surprising consequence if EVERYONE SHRANK TO 12 INCHES TALL?\"]\n",
        "typeref = ['uses', 'instances', 'consequences', 'uses', 'instances', 'consequences']\n",
        "qref_dict = dict(zip(promptref, qref))\n",
        "\n",
        "df, meta = pyreadstat.read_sav('/content/DT_Responses_PACA_2008_Study_2.sav')\n",
        "df = df.rename(columns={'subject':'participant', 'order':'response_num'})\n",
        "df['prompt'] = df.task.apply(lambda x: promptref[int(x)-1])\n",
        "df['type'] = df.task.apply(lambda x: typeref[int(x)-1])\n",
        "df['participant'] = df['participant'].astype(int)\n",
        "df['question'] = df.prompt.apply(lambda x: qref_dict[x])\n",
        "df = prep_general(df, src, include_stats=True, return_full=True,\n",
        "                  include_rater_std=include_rater_std)\n",
        "# doublecheck - burczak reported ICC2k as 0.48 (for the s)\n",
        "print(\"Testing stats just for AUT (burczak et al reported ICC2k as 0.48)\")\n",
        "prep_general(df.query(\"type=='uses'\"), src, include_stats=True)\n",
        "datasets[src] = df[returncols]\n",
        "datasets[src].sample(2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UH3yOwIon4Ky"
      },
      "source": [
        "## Process Hofelich Mohr, Sell, and Lindsay\n",
        "\n",
        "Doublecheck ICC2k - buczak paper had icc2k=0.67"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JHgvPMmynu6i"
      },
      "outputs": [],
      "source": [
        "!wget -qO hofelich.zip https://conservancy.umn.edu/bitstream/handle/11299/172116/HMSL_CSV%20Data%20Files.zip?sequence=28&isAllowed=y\n",
        "!unzip -qo hofelich.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BaeZVSf_oCSI",
        "outputId": "b07e9f80-09c3-4a30-b447-93ffda9b4a90"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Rater cols: ['J1_Rating', 'J2_Rating', 'J3_Rating', 'J4_Rating']\n",
            "Dropping 23 unrated items\n",
            "assuming type=\"uses\"\n",
            "# of prompts 2\n",
            "# of participants 638\n",
            "# of data points 3843\n",
            "Prompts ['paperclip' 'brick']\n",
            "# of raters 4\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-1-2ab89a7cc4a3>:51: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data['src'] = src\n",
            "<ipython-input-1-2ab89a7cc4a3>:52: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data['avg_rating'] = data[rater_cols].mean(1)\n",
            "<ipython-input-1-2ab89a7cc4a3>:60: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data['target'] = normalize_values(data.avg_rating, oldrange=inputrange)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F   df1    df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.30  2.71  3713  11142   0.0  [0.28, 0.32]\n",
              "1   ICC2     Single random raters  0.33  3.86  3713  11139   0.0  [0.22, 0.43]\n",
              "2   ICC3      Single fixed raters  0.42  3.86  3713  11139   0.0   [0.4, 0.43]\n",
              "3  ICC1k  Average raters absolute  0.63  2.71  3713  11142   0.0  [0.61, 0.65]\n",
              "4  ICC2k    Average random raters  0.67  3.86  3713  11139   0.0  [0.53, 0.75]\n",
              "5  ICC3k     Average fixed raters  0.74  3.86  3713  11139   0.0  [0.73, 0.75]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4ceee798-9b94-4ddc-82ab-e07ea9faffee\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.30</td>\n",
              "      <td>2.71</td>\n",
              "      <td>3713</td>\n",
              "      <td>11142</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.28, 0.32]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.33</td>\n",
              "      <td>3.86</td>\n",
              "      <td>3713</td>\n",
              "      <td>11139</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.22, 0.43]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.42</td>\n",
              "      <td>3.86</td>\n",
              "      <td>3713</td>\n",
              "      <td>11139</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.4, 0.43]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.63</td>\n",
              "      <td>2.71</td>\n",
              "      <td>3713</td>\n",
              "      <td>11142</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.61, 0.65]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.67</td>\n",
              "      <td>3.86</td>\n",
              "      <td>3713</td>\n",
              "      <td>11139</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.53, 0.75]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.74</td>\n",
              "      <td>3.86</td>\n",
              "      <td>3713</td>\n",
              "      <td>11139</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.73, 0.75]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4ceee798-9b94-4ddc-82ab-e07ea9faffee')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4ceee798-9b94-4ddc-82ab-e07ea9faffee button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4ceee798-9b94-4ddc-82ab-e07ea9faffee');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      type   src                                   question     prompt  \\\n",
              "787   uses  hmsl  What is a surprising use for a PAPERCLIP?  paperclip   \n",
              "1076  uses  hmsl      What is a surprising use for a BRICK?      brick   \n",
              "\n",
              "       response                     id  target   participant  response_num  \n",
              "787      Weapon  hmsl_paperclip-db4a2a     1.5  hmsluirMt5sS           1.0  \n",
              "1076  door stop      hmsl_brick-eae6a3     1.2  hmsl68P7GeOl           1.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ed7666c0-9635-4e84-aeef-b6de1774b0d3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>787</th>\n",
              "      <td>uses</td>\n",
              "      <td>hmsl</td>\n",
              "      <td>What is a surprising use for a PAPERCLIP?</td>\n",
              "      <td>paperclip</td>\n",
              "      <td>Weapon</td>\n",
              "      <td>hmsl_paperclip-db4a2a</td>\n",
              "      <td>1.5</td>\n",
              "      <td>hmsluirMt5sS</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1076</th>\n",
              "      <td>uses</td>\n",
              "      <td>hmsl</td>\n",
              "      <td>What is a surprising use for a BRICK?</td>\n",
              "      <td>brick</td>\n",
              "      <td>door stop</td>\n",
              "      <td>hmsl_brick-eae6a3</td>\n",
              "      <td>1.2</td>\n",
              "      <td>hmsl68P7GeOl</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ed7666c0-9635-4e84-aeef-b6de1774b0d3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ed7666c0-9635-4e84-aeef-b6de1774b0d3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ed7666c0-9635-4e84-aeef-b6de1774b0d3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "src = 'hmsl'\n",
        "df = pd.read_csv(f'HMSL_Originality_scores_all.csv')\n",
        "df = df.rename(columns={'Item': 'prompt', 'QLogin_1':'participant'})\n",
        "rater_cols = ['J1_Rating','J2_Rating','J3_Rating','J4_Rating']\n",
        "# remove three ratings that are 11\n",
        "df[rater_cols] = df[rater_cols].replace(11, pd.NA)\n",
        "datasets[src] = prep_general(df, src, rater_cols=rater_cols, include_rater_std=include_rater_std)\n",
        "datasets[src].sample(2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VgNoUTqg_dQb"
      },
      "source": [
        "## Process dataset used by Beaty and Johnson\n",
        "\n",
        "From SemDis paper:\n",
        "\n",
        "- Study 1 was re-analysis of AUT responses from Beaty et al., 2018 to see if ensemble approaches work better. Two tests: `box` and `rope`\n",
        "   - according to their paper, using additive composition was slightly negative correlation, while multiplicative 'results revealed a large correlation between latent semantic distance and human ratings:$r=.91$, p<.001'. This uses a model that weighs the factors, but is (I think) tailored to the dataset without held out data.\n",
        "\n",
        "- Study 2 was re-analysis of results from Silvia et al. 2017, also on box and rope\n",
        "- Study 3 was brick - yet again - via Beaty and Silvia 2012\n",
        "- Study 4 and 5- Heinen and Johnson (2018) - were noun matching, not relevant here"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a4b_fi1y_SdN"
      },
      "outputs": [],
      "source": [
        "!wget -qO beaty.zip https://files.osf.io/v1/resources/gz4fc/providers/osfstorage/5e45b6c73e86a800be6e662e/?zip=\n",
        "!unzip -qo beaty.zip\n",
        "files = sorted(list(Path('.').glob('Study*/*data_long.xlsx')))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v3xBdaTnSktv",
        "outputId": "282c1875-5a7a-4fab-f975-b03ca51a421d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-----------BETAL18------------\n",
            "Rater cols: ['rater1', 'rater2', 'rater3', 'rater4']\n",
            "assuming type=\"uses\"\n",
            "# of prompts 2\n",
            "# of participants 171\n",
            "# of data points 2918\n",
            "Prompts ['box' 'rope']\n",
            "# of raters 4\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F   df1   df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.52  5.26  2894  8685   0.0   [0.5, 0.53]\n",
              "1   ICC2     Single random raters  0.52  6.18  2894  8682   0.0  [0.46, 0.58]\n",
              "2   ICC3      Single fixed raters  0.56  6.18  2894  8682   0.0  [0.55, 0.58]\n",
              "3  ICC1k  Average raters absolute  0.81  5.26  2894  8685   0.0   [0.8, 0.82]\n",
              "4  ICC2k    Average random raters  0.82  6.18  2894  8682   0.0  [0.77, 0.85]\n",
              "5  ICC3k     Average fixed raters  0.84  6.18  2894  8682   0.0  [0.83, 0.85]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-71a8d8a2-4230-4826-a7cc-fa23f62d9be1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.52</td>\n",
              "      <td>5.26</td>\n",
              "      <td>2894</td>\n",
              "      <td>8685</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.5, 0.53]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.52</td>\n",
              "      <td>6.18</td>\n",
              "      <td>2894</td>\n",
              "      <td>8682</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.46, 0.58]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.56</td>\n",
              "      <td>6.18</td>\n",
              "      <td>2894</td>\n",
              "      <td>8682</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.55, 0.58]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.81</td>\n",
              "      <td>5.26</td>\n",
              "      <td>2894</td>\n",
              "      <td>8685</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.8, 0.82]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.82</td>\n",
              "      <td>6.18</td>\n",
              "      <td>2894</td>\n",
              "      <td>8682</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.77, 0.85]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.84</td>\n",
              "      <td>6.18</td>\n",
              "      <td>2894</td>\n",
              "      <td>8682</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.83, 0.85]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-71a8d8a2-4230-4826-a7cc-fa23f62d9be1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-71a8d8a2-4230-4826-a7cc-fa23f62d9be1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-71a8d8a2-4230-4826-a7cc-fa23f62d9be1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "------------SNB17-------------\n",
            "Rater cols: ['rater1', 'rater2', 'rater3']\n",
            "assuming type=\"uses\"\n",
            "# of prompts 2\n",
            "# of participants 142\n",
            "# of data points 2372\n",
            "Prompts ['box' 'rope']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F   df1   df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.38  2.82  2352  4706   0.0   [0.35, 0.4]\n",
              "1   ICC2     Single random raters  0.40  3.57  2352  4704   0.0   [0.29, 0.5]\n",
              "2   ICC3      Single fixed raters  0.46  3.57  2352  4704   0.0  [0.44, 0.49]\n",
              "3  ICC1k  Average raters absolute  0.65  2.82  2352  4706   0.0  [0.62, 0.67]\n",
              "4  ICC2k    Average random raters  0.67  3.57  2352  4704   0.0  [0.55, 0.75]\n",
              "5  ICC3k     Average fixed raters  0.72  3.57  2352  4704   0.0   [0.7, 0.74]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7b18f8d8-7d8a-49c3-80a3-e61b09163aec\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.38</td>\n",
              "      <td>2.82</td>\n",
              "      <td>2352</td>\n",
              "      <td>4706</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.35, 0.4]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.40</td>\n",
              "      <td>3.57</td>\n",
              "      <td>2352</td>\n",
              "      <td>4704</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.29, 0.5]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.46</td>\n",
              "      <td>3.57</td>\n",
              "      <td>2352</td>\n",
              "      <td>4704</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.44, 0.49]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.65</td>\n",
              "      <td>2.82</td>\n",
              "      <td>2352</td>\n",
              "      <td>4706</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.62, 0.67]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.67</td>\n",
              "      <td>3.57</td>\n",
              "      <td>2352</td>\n",
              "      <td>4704</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.55, 0.75]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.72</td>\n",
              "      <td>3.57</td>\n",
              "      <td>2352</td>\n",
              "      <td>4704</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.7, 0.74]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7b18f8d8-7d8a-49c3-80a3-e61b09163aec')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7b18f8d8-7d8a-49c3-80a3-e61b09163aec button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7b18f8d8-7d8a-49c3-80a3-e61b09163aec');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------BS12-------------\n",
            "Rater cols: ['br_rater1', 'br_rater2', 'br_rater3']\n",
            "assuming type=\"uses\"\n",
            "# of prompts 1\n",
            "# of participants 133\n",
            "# of data points 1807\n",
            "Prompts ['brick']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F   df1   df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.43  3.24  1800  3602   0.0   [0.4, 0.46]\n",
              "1   ICC2     Single random raters  0.46  4.46  1800  3600   0.0   [0.3, 0.57]\n",
              "2   ICC3      Single fixed raters  0.54  4.46  1800  3600   0.0  [0.51, 0.56]\n",
              "3  ICC1k  Average raters absolute  0.69  3.24  1800  3602   0.0  [0.67, 0.72]\n",
              "4  ICC2k    Average random raters  0.72  4.46  1800  3600   0.0   [0.56, 0.8]\n",
              "5  ICC3k     Average fixed raters  0.78  4.46  1800  3600   0.0  [0.76, 0.79]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0b4a0fb6-c9c8-41e2-b223-a826a4933385\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.43</td>\n",
              "      <td>3.24</td>\n",
              "      <td>1800</td>\n",
              "      <td>3602</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.4, 0.46]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.46</td>\n",
              "      <td>4.46</td>\n",
              "      <td>1800</td>\n",
              "      <td>3600</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.3, 0.57]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.54</td>\n",
              "      <td>4.46</td>\n",
              "      <td>1800</td>\n",
              "      <td>3600</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.51, 0.56]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.69</td>\n",
              "      <td>3.24</td>\n",
              "      <td>1800</td>\n",
              "      <td>3602</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.67, 0.72]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.72</td>\n",
              "      <td>4.46</td>\n",
              "      <td>1800</td>\n",
              "      <td>3600</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.56, 0.8]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.78</td>\n",
              "      <td>4.46</td>\n",
              "      <td>1800</td>\n",
              "      <td>3600</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.76, 0.79]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0b4a0fb6-c9c8-41e2-b223-a826a4933385')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0b4a0fb6-c9c8-41e2-b223-a826a4933385 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0b4a0fb6-c9c8-41e2-b223-a826a4933385');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "for fname,src in zip(files[:3], ['betal18','snb17', 'bs12']):\n",
        "    print(src.upper().center(30, '-'))\n",
        "    df = pd.read_excel(fname)\n",
        "\n",
        "    df = df.rename(columns={'id':'participant', 'item':'prompt'})\n",
        "    df['response_num'] = None\n",
        "    datasets[src] = prep_general(df, src, include_rater_std=include_rater_std)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "laa40OuiNV2X"
      },
      "source": [
        "### Doublecheck Beaty data\n",
        "Confirm that this data is what's seen in table 1 of paper - this is correct."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kl7M4fLvLzt6",
        "outputId": "fd68d1a6-be51-435c-cd5a-a99b1f525d5f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          b_rater1  b_rater2  b_rater3  b_rater4  b_cbowu  b_cbows  b_cboww  \\\n",
              "b_rater1      1.00      0.65      0.54      0.73     0.46     0.32     0.43   \n",
              "b_rater2      0.65      1.00      0.59      0.74     0.48     0.37     0.45   \n",
              "b_rater3      0.54      0.59      1.00      0.73     0.38     0.45     0.36   \n",
              "b_rater4      0.73      0.74      0.73      1.00     0.45     0.38     0.39   \n",
              "b_cbowu       0.46      0.48      0.38      0.45     1.00     0.76     0.84   \n",
              "b_cbows       0.32      0.37      0.45      0.38     0.76     1.00     0.69   \n",
              "b_cboww       0.43      0.45      0.36      0.39     0.84     0.69     1.00   \n",
              "b_tasa        0.30      0.27      0.22      0.22     0.36     0.32     0.41   \n",
              "b_glov        0.26      0.24      0.21      0.26     0.80     0.57     0.71   \n",
              "r_rater1      0.29      0.38      0.37      0.43     0.17     0.10     0.09   \n",
              "r_rater2      0.27      0.38      0.39      0.41     0.17     0.13     0.10   \n",
              "r_rater3      0.36      0.48      0.51      0.54     0.31     0.23     0.20   \n",
              "r_rater4      0.37      0.43      0.48      0.51     0.31     0.21     0.21   \n",
              "r_cbowu       0.37      0.40      0.36      0.49     0.42     0.35     0.41   \n",
              "r_cbows       0.32      0.42      0.39      0.45     0.39     0.34     0.40   \n",
              "r_cboww       0.36      0.43      0.37      0.50     0.48     0.44     0.46   \n",
              "r_tasa        0.25      0.35      0.29      0.37     0.28     0.19     0.17   \n",
              "r_glov        0.37      0.42      0.35      0.45     0.43     0.40     0.45   \n",
              "\n",
              "          b_tasa  b_glov  r_rater1  r_rater2  r_rater3  r_rater4  r_cbowu  \\\n",
              "b_rater1    0.30    0.26      0.29      0.27      0.36      0.37     0.37   \n",
              "b_rater2    0.27    0.24      0.38      0.38      0.48      0.43     0.40   \n",
              "b_rater3    0.22    0.21      0.37      0.39      0.51      0.48     0.36   \n",
              "b_rater4    0.22    0.26      0.43      0.41      0.54      0.51     0.49   \n",
              "b_cbowu     0.36    0.80      0.17      0.17      0.31      0.31     0.42   \n",
              "b_cbows     0.32    0.57      0.10      0.13      0.23      0.21     0.35   \n",
              "b_cboww     0.41    0.71      0.09      0.10      0.20      0.21     0.41   \n",
              "b_tasa      1.00    0.31     -0.05      0.06      0.08      0.10     0.10   \n",
              "b_glov      0.31    1.00      0.07      0.08      0.19      0.22     0.37   \n",
              "r_rater1   -0.05    0.07      1.00      0.65      0.74      0.74     0.32   \n",
              "r_rater2    0.06    0.08      0.65      1.00      0.75      0.76     0.38   \n",
              "r_rater3    0.08    0.19      0.74      0.75      1.00      0.87     0.57   \n",
              "r_rater4    0.10    0.22      0.74      0.76      0.87      1.00     0.54   \n",
              "r_cbowu     0.10    0.37      0.32      0.38      0.57      0.54     1.00   \n",
              "r_cbows     0.13    0.31      0.33      0.38      0.54      0.54     0.84   \n",
              "r_cboww     0.15    0.39      0.32      0.34      0.53      0.51     0.90   \n",
              "r_tasa      0.02    0.22      0.33      0.35      0.54      0.47     0.66   \n",
              "r_glov      0.12    0.33      0.29      0.33      0.50      0.50     0.88   \n",
              "\n",
              "          r_cbows  r_cboww  r_tasa  r_glov  \n",
              "b_rater1     0.32     0.36    0.25    0.37  \n",
              "b_rater2     0.42     0.43    0.35    0.42  \n",
              "b_rater3     0.39     0.37    0.29    0.35  \n",
              "b_rater4     0.45     0.50    0.37    0.45  \n",
              "b_cbowu      0.39     0.48    0.28    0.43  \n",
              "b_cbows      0.34     0.44    0.19    0.40  \n",
              "b_cboww      0.40     0.46    0.17    0.45  \n",
              "b_tasa       0.13     0.15    0.02    0.12  \n",
              "b_glov       0.31     0.39    0.22    0.33  \n",
              "r_rater1     0.33     0.32    0.33    0.29  \n",
              "r_rater2     0.38     0.34    0.35    0.33  \n",
              "r_rater3     0.54     0.53    0.54    0.50  \n",
              "r_rater4     0.54     0.51    0.47    0.50  \n",
              "r_cbowu      0.84     0.90    0.66    0.88  \n",
              "r_cbows      1.00     0.87    0.61    0.81  \n",
              "r_cboww      0.87     1.00    0.60    0.87  \n",
              "r_tasa       0.61     0.60    1.00    0.59  \n",
              "r_glov       0.81     0.87    0.59    1.00  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-85f867d8-3e28-48be-af2e-ae1396d92ac2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>b_rater1</th>\n",
              "      <th>b_rater2</th>\n",
              "      <th>b_rater3</th>\n",
              "      <th>b_rater4</th>\n",
              "      <th>b_cbowu</th>\n",
              "      <th>b_cbows</th>\n",
              "      <th>b_cboww</th>\n",
              "      <th>b_tasa</th>\n",
              "      <th>b_glov</th>\n",
              "      <th>r_rater1</th>\n",
              "      <th>r_rater2</th>\n",
              "      <th>r_rater3</th>\n",
              "      <th>r_rater4</th>\n",
              "      <th>r_cbowu</th>\n",
              "      <th>r_cbows</th>\n",
              "      <th>r_cboww</th>\n",
              "      <th>r_tasa</th>\n",
              "      <th>r_glov</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>b_rater1</th>\n",
              "      <td>1.00</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.73</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.30</td>\n",
              "      <td>0.26</td>\n",
              "      <td>0.29</td>\n",
              "      <td>0.27</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.25</td>\n",
              "      <td>0.37</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_rater2</th>\n",
              "      <td>0.65</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.59</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.27</td>\n",
              "      <td>0.24</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.42</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.42</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_rater3</th>\n",
              "      <td>0.54</td>\n",
              "      <td>0.59</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.73</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.29</td>\n",
              "      <td>0.35</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_rater4</th>\n",
              "      <td>0.73</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.73</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.26</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.45</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_cbowu</th>\n",
              "      <td>0.46</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.45</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.84</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.80</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.42</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.28</td>\n",
              "      <td>0.43</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_cbows</th>\n",
              "      <td>0.32</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.76</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.69</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.23</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.34</td>\n",
              "      <td>0.44</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.40</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_cboww</th>\n",
              "      <td>0.43</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.84</td>\n",
              "      <td>0.69</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.71</td>\n",
              "      <td>0.09</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.45</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_tasa</th>\n",
              "      <td>0.30</td>\n",
              "      <td>0.27</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.41</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.31</td>\n",
              "      <td>-0.05</td>\n",
              "      <td>0.06</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.15</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_glov</th>\n",
              "      <td>0.26</td>\n",
              "      <td>0.24</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.26</td>\n",
              "      <td>0.80</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.71</td>\n",
              "      <td>0.31</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_rater1</th>\n",
              "      <td>0.29</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.09</td>\n",
              "      <td>-0.05</td>\n",
              "      <td>0.07</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_rater2</th>\n",
              "      <td>0.27</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.06</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.65</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.75</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.34</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_rater3</th>\n",
              "      <td>0.36</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.23</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.75</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.53</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_rater4</th>\n",
              "      <td>0.37</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.87</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_cbowu</th>\n",
              "      <td>0.37</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.42</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.54</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.84</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_cbows</th>\n",
              "      <td>0.32</td>\n",
              "      <td>0.42</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.34</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.84</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.81</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_cboww</th>\n",
              "      <td>0.36</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.44</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.15</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.34</td>\n",
              "      <td>0.53</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0.87</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.87</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_tasa</th>\n",
              "      <td>0.25</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.29</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.28</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.60</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_glov</th>\n",
              "      <td>0.37</td>\n",
              "      <td>0.42</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.12</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.29</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.88</td>\n",
              "      <td>0.81</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.59</td>\n",
              "      <td>1.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-85f867d8-3e28-48be-af2e-ae1396d92ac2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-85f867d8-3e28-48be-af2e-ae1396d92ac2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-85f867d8-3e28-48be-af2e-ae1396d92ac2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "test = pd.read_excel('/content/Study 1/s1_data_agg.xlsx')\n",
        "test[[col for col in test if (col.startswith('b_') or col.startswith('r_')) and not col.endswith('_a')]].corr().round(2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8H-WPj9MeGrS"
      },
      "source": [
        "## Normalize MOTES Pilot Data\n",
        "\n",
        "Score on multi-point scale.\n",
        "\n",
        "MOTES is related to the \"Measuring Original Thinking in Elementary Students: A Text-Mining Approach\" (IES #R305A200519). This dataset is very, very new and not yet ready to share. Hopefully it is open access by the time our first LLM paper is published, until then I think we can share with researchers by request (contact <peter.organisciak@du.edu> and/or <selcuk.acar@unt.edu>)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9d5mIJsYQ7oD",
        "outputId": "ff237725-162d-4dad-8344-e620c245113d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Rater cols: ['rater1', 'rater2', 'rater3']\n",
            "# of prompts 10\n",
            "# of participants 35\n",
            "# of data points 339\n",
            "Prompts ['backpack' 'ball' 'bottle' 'hat' 'lightbulb' 'pencil' 'shoe' 'sock'\n",
            " 'spoon' 'toothbrush']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F  df1  df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.58  5.15  338  678   0.0  [0.52, 0.63]\n",
              "1   ICC2     Single random raters  0.58  5.28  338  676   0.0  [0.52, 0.64]\n",
              "2   ICC3      Single fixed raters  0.59  5.28  338  676   0.0  [0.53, 0.64]\n",
              "3  ICC1k  Average raters absolute  0.81  5.15  338  678   0.0  [0.77, 0.84]\n",
              "4  ICC2k    Average random raters  0.81  5.28  338  676   0.0  [0.77, 0.84]\n",
              "5  ICC3k     Average fixed raters  0.81  5.28  338  676   0.0  [0.77, 0.84]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b5fb5cdc-5725-4527-9eda-d3c720f63e68\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.58</td>\n",
              "      <td>5.15</td>\n",
              "      <td>338</td>\n",
              "      <td>678</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.52, 0.63]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.58</td>\n",
              "      <td>5.28</td>\n",
              "      <td>338</td>\n",
              "      <td>676</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.52, 0.64]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.59</td>\n",
              "      <td>5.28</td>\n",
              "      <td>338</td>\n",
              "      <td>676</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.53, 0.64]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.81</td>\n",
              "      <td>5.15</td>\n",
              "      <td>338</td>\n",
              "      <td>678</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.77, 0.84]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.81</td>\n",
              "      <td>5.28</td>\n",
              "      <td>338</td>\n",
              "      <td>676</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.77, 0.84]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.81</td>\n",
              "      <td>5.28</td>\n",
              "      <td>338</td>\n",
              "      <td>676</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.77, 0.84]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b5fb5cdc-5725-4527-9eda-d3c720f63e68')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b5fb5cdc-5725-4527-9eda-d3c720f63e68 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b5fb5cdc-5725-4527-9eda-d3c720f63e68');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "src = 'motesp'\n",
        "motes_pilot = pd.read_csv(os.path.join(base_dir, 'motes_pilot_gt_scores.csv')).rename(columns=dict(D='rater1', K='rater2', T='rater3', ID='participant'))\n",
        "motes_pilot['response_num'] = None\n",
        "motes_pilot['type'] = motes_pilot.task.apply(lambda x: x.split('_')[0]).replace({'g1':'uses', 'g2':'instances', 'g3': 'consequences', 'g4':'completion'})\n",
        "motes_pilot = prep_general(motes_pilot, src, include_stats=False, return_full=True, include_rater_std=include_rater_std)\n",
        "# Normalize question styles\n",
        "motes_pilot.question = (motes_pilot.question.str.replace(\"What would be a surprising\", \"What is a surprising\", regex=True)\n",
        "                            .str.replace(\"Think of an example of something that is (.*)\", \"What is a surprising example of something \\\\1?\", regex=True)\n",
        "                            .str.replace(\"What would happen if (.*)\", \"What would be a surprising consequence if \\\\1\", regex=True)\n",
        ")\n",
        "comprows = (motes_pilot.type == 'completion')\n",
        "motes_pilot.loc[comprows, 'question'] = motes_pilot.loc[comprows, 'question'].str.replace(\"^(.*)$\", 'Complete this sentence in a surprising way: \"\\\\1...\"', regex=True)\n",
        "datasets[src] = motes_pilot[returncols]\n",
        "\n",
        "simple_stats(motes_pilot[motes_pilot.id.str.contains('_g1_')], ['rater1', 'rater2', 'rater3'])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tbICpoGS4nvR"
      },
      "source": [
        "## Process MOTES Main Data\n",
        "\n",
        "This is the post-pilot data. As with the pilot data, this dataset is not yet released, but please reach out.\n",
        "\n",
        "There are currently 5 raters, though early analysis had 4, which meant the randomized rounding strategy sometimes went into effect - e.g. 2.25 might become 2.3 or 2.2."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9l-yEQEQ1p2f"
      },
      "outputs": [],
      "source": [
        "df = pd.read_csv(os.path.join(base_dir, 'motes_full_gt_scores.csv')).replace(-999, pd.NA).copy()\n",
        "df['participant'] = df.Order.astype(str).apply(lambda x: hashlib.md5(x.encode('utf-8')).hexdigest()[:6])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dDWDbuTf94vS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e3e783d6-cfaf-4786-cb6c-00d1232d9fc1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Rater cols: ['Kscore', 'Hscore', 'Cscore', 'Tscore', 'Mscore']\n",
            "# of prompts 24\n",
            "# of participants 386\n",
            "# of data points 8563\n",
            "Prompts ['ball' 'sock' 'pencil' 'spoon' 'lightbulb' 'hat' 'bottle' 'toothbrush'\n",
            " 'smelly' 'soft' 'red' 'frozen' 'wet' 'huge' 'fun' 'tasty' 'school bus'\n",
            " 'games' 'library' 'lecture' 'phone' 'rain' 'closet' 'lunchroom']\n",
            "# of raters 5\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-1-2ab89a7cc4a3>:51: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data['src'] = src\n",
            "<ipython-input-1-2ab89a7cc4a3>:52: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data['avg_rating'] = data[rater_cols].mean(1)\n",
            "<ipython-input-1-2ab89a7cc4a3>:60: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data['target'] = normalize_values(data.avg_rating, oldrange=inputrange)\n",
            "<ipython-input-1-2ab89a7cc4a3>:67: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data['participant'] = src + data['participant'].astype(str)\n",
            "<ipython-input-1-2ab89a7cc4a3>:82: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data['id'] = f'{src}_' + data['task' if 'task' in data.columns else 'prompt'].astype(str) + '-' + idhash\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Type              Description   ICC     F   df1    df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.40  4.34  8559  34240   0.0  [0.39, 0.41]\n",
              "1   ICC2     Single random raters  0.42  6.41  8559  34236   0.0   [0.3, 0.52]\n",
              "2   ICC3      Single fixed raters  0.52  6.41  8559  34236   0.0  [0.51, 0.53]\n",
              "3  ICC1k  Average raters absolute  0.77  4.34  8559  34240   0.0  [0.76, 0.78]\n",
              "4  ICC2k    Average random raters  0.79  6.41  8559  34236   0.0  [0.69, 0.85]\n",
              "5  ICC3k     Average fixed raters  0.84  6.41  8559  34236   0.0  [0.84, 0.85]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d390f565-4245-46ce-a0e0-565b4b90c276\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.40</td>\n",
              "      <td>4.34</td>\n",
              "      <td>8559</td>\n",
              "      <td>34240</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.39, 0.41]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.42</td>\n",
              "      <td>6.41</td>\n",
              "      <td>8559</td>\n",
              "      <td>34236</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.3, 0.52]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.52</td>\n",
              "      <td>6.41</td>\n",
              "      <td>8559</td>\n",
              "      <td>34236</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.51, 0.53]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.77</td>\n",
              "      <td>4.34</td>\n",
              "      <td>8559</td>\n",
              "      <td>34240</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.76, 0.78]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.79</td>\n",
              "      <td>6.41</td>\n",
              "      <td>8559</td>\n",
              "      <td>34236</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.69, 0.85]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.84</td>\n",
              "      <td>6.41</td>\n",
              "      <td>8559</td>\n",
              "      <td>34236</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.84, 0.85]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d390f565-4245-46ce-a0e0-565b4b90c276')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d390f565-4245-46ce-a0e0-565b4b90c276 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d390f565-4245-46ce-a0e0-565b4b90c276');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            type     src                                           question  \\\n",
              "7706  completion  motesf  Complete this sentence in a surprising way: \"I...   \n",
              "\n",
              "     prompt                           response                  id  target  \\\n",
              "7706   rain  I saw fish come down from the sky  motesf_rain-92569d     4.2   \n",
              "\n",
              "       participant response_num  \n",
              "7706  motesf2723d0            2  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2291cf6b-0cdf-4720-9a4f-5eb972dd69d5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>7706</th>\n",
              "      <td>completion</td>\n",
              "      <td>motesf</td>\n",
              "      <td>Complete this sentence in a surprising way: \"I...</td>\n",
              "      <td>rain</td>\n",
              "      <td>I saw fish come down from the sky</td>\n",
              "      <td>motesf_rain-92569d</td>\n",
              "      <td>4.2</td>\n",
              "      <td>motesf2723d0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2291cf6b-0cdf-4720-9a4f-5eb972dd69d5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2291cf6b-0cdf-4720-9a4f-5eb972dd69d5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2291cf6b-0cdf-4720-9a4f-5eb972dd69d5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "src = 'motesf'\n",
        "\n",
        "corrected = True # use spelling corrected columns\n",
        "items = [col.replace('_prompt', '') for col in df.columns if col.startswith('G') and col.endswith('_prompt')]\n",
        "\n",
        "collector = []\n",
        "# RESHAPE TO long\n",
        "for item in items:\n",
        "    subset = df[['participant'] + [col for col in df.columns if col.startswith(item)]].copy()\n",
        "    subset.columns = [col.split('_')[-1] for col in subset.columns]\n",
        "    subset['game'] = item.split('_')[0]\n",
        "    subset['prompt_code'] = item\n",
        "    collector.append(subset)\n",
        "reshaped = pd.concat(collector)\n",
        "# remove non-responses\n",
        "reshaped = reshaped[~reshaped.raw.isna()]\n",
        "# restore original wording in the test\n",
        "reshaped.prompt =reshaped.prompt.str.replace('light bulbs', 'lightbulb').str.replace('hat cap', 'hat').str.replace('soccer ball', 'ball').str.replace('lead pencil', 'pencil').str.replace('spoons', 'spoon')\n",
        "\n",
        "# add display order\n",
        "displayorder = df[['participant'] + [col for col in df.columns if 'DO' in col]]\n",
        "displayorder = displayorder.melt(id_vars='participant', value_name='prompt_code')\n",
        "displayorder['response_num'] = displayorder.variable.apply(lambda x:x[-1])\n",
        "reshaped = reshaped.merge(displayorder[['participant', 'prompt_code', 'response_num']])\n",
        "# use spelling corrected response, unless set otherwise\n",
        "reshaped = reshaped.rename(columns={('corrected' if corrected else 'raw'):'response'})\n",
        "reshaped['type'] = reshaped['game'].replace({'G1':'uses', 'G2': 'instances', 'G3':'completion'})\n",
        "\n",
        "completion_ref = {\n",
        "    \"playground\": \"When the friends met on the playground...\",\n",
        "    \"school bus\": \"When I got on the school bus...\",\n",
        "    \"games\": \"At a sleepover we...\",\n",
        "    'library': \"When the kids were in the library...\",\n",
        "    'lecture': \"When the teacher was talking...\",\n",
        "    'phone': \"My friend called me on the phone to tell me...\",\n",
        "    'rain': \"It started raining and...\",\n",
        "    'closet': \"When I opened my closet...\",\n",
        "    'lunchroom': \"When I was at lunch...\"\n",
        "}\n",
        "reshaped.loc[reshaped.type == 'uses', 'question'] = reshaped.loc[reshaped.type == 'uses', 'prompt'].apply(lambda x: f\"What is a surprising use for a {x.upper()}?\")\n",
        "reshaped.loc[reshaped.type == 'instances','question'] = reshaped.loc[reshaped.type == 'instances', 'prompt'].apply(lambda x: f\"What is a surprising example of something {x.upper()}?\")\n",
        "reshaped.loc[(reshaped.type == 'completion'), 'question'] = reshaped.loc[reshaped.type == 'completion', 'prompt'].replace(completion_ref).str.replace(\"(.*)\", 'Complete this sentence in a surprising way: \"\\\\1\"...', regex=True)\n",
        "\n",
        "datasets[src] = prep_general(reshaped, src,\n",
        "                             rater_cols=[col for col in reshaped if 'score' in col.lower()],\n",
        "                             include_rater_std=include_rater_std, inputrange=(1,5))\n",
        "\n",
        "# This is code for a separate purpose - exporting spell corrected and uncorrected data\n",
        "if False:\n",
        "    #corrected_res = datasets[src]\n",
        "    #uncorrected = datasets[src]\n",
        "    participant_info = df[['Grade', 'Age at MOTES', 'participant']]\n",
        "    participant_info.columns = ['grade', 'age', 'participant']\n",
        "    participant_info.grade = participant_info.grade.apply(lambda x: int(x[0]))\n",
        "    participant_info.participant = src + participant_info.participant\n",
        "    spelldata = uncorrected.rename(columns={'response':'raw'}).merge(corrected_res.rename(columns={'response':'corrected'}), on=[col for col in uncorrected.columns if col not in ['id', 'response']]).merge(participant_info)\n",
        "    spelldata.to_csv(base_dir / 'motes_full_spelldata.csv')\n",
        "\n",
        "datasets[src].sample()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9mbZjzsjIuW8"
      },
      "source": [
        "## Combine data and basic stats"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HjzAueLF-G2j"
      },
      "source": [
        "Check correlation among ratings for responses which have been submitted more than once. Here, I sample one rating vs mean of all the ratings for a duplicate response. This contextualizes the max what a model might be able to do - if humans can't agree (sometimes with themselves!), then it would be impossible for a model to do so."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown Average rating-to-rating correlation on duplicates\n",
        "# run multiple times with different samples\n",
        "from tqdm import trange\n",
        "corrs = []\n",
        "for i in trange(1000):\n",
        "    check_dupe_corr = pd.concat(datasets.values()).sample(frac=1, random_state=i**2)\n",
        "    just_duped = check_dupe_corr[check_dupe_corr[['prompt', 'response']].duplicated(keep=False)]\n",
        "    first = just_duped.drop_duplicates(['prompt', 'response'], keep='first')\n",
        "    last = just_duped.drop_duplicates(['prompt', 'response'], keep='last')\n",
        "    merged = first.merge(last[['prompt', 'response', 'target']], how='inner', on=['prompt', 'response'])\n",
        "    corr = merged[['target_x', 'target_y']].corr().values[0,1]\n",
        "    corrs.append(corr)\n",
        "print(\"\\nAverage correlation among duplicates\", sum(corrs)/len(corrs))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form",
        "id": "zHONheN0k0Q4",
        "outputId": "b2e4ee80-602c-4139-e50b-77a6581d7aa5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1000/1000 [01:04<00:00, 15.53it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Average correlation among duplicates 0.8757464408564409\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dM-KuLyz0gDx",
        "outputId": "445a2f2c-dc11-465f-ab3f-90000dd530e9",
        "cellView": "form"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1000/1000 [01:17<00:00, 12.88it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Average correlation among duplicates 0.9100034032038847\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "#@markdown Average rating-to-mean(other ratings) correlation on duplicates\n",
        "corrs = []\n",
        "for i in trange(1000):\n",
        "    check_dupe_corr = pd.concat(datasets.values()).sample(frac=1)\n",
        "    means_of_dupes = check_dupe_corr[check_dupe_corr[['prompt', 'response']].duplicated()].groupby(['prompt','response'], as_index=False).target.mean().round(2)\n",
        "    corr = check_dupe_corr.merge(means_of_dupes[['prompt', 'response', 'target']], on=['prompt', 'response']).corr().loc['target_x', 'target_y']\n",
        "    corrs.append(corr)\n",
        "print(\"\\nAverage correlation among duplicates\", sum(corrs)/len(corrs))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fTgfgBWR_c-H"
      },
      "source": [
        "Merge ratings for items with duplicates, so that a response that has been rated multiple times has the average of all instances as it's ground truth."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 520
        },
        "id": "b0hHuZqDpCX_",
        "outputId": "67fa8b4c-6400-4f1f-a62a-b22e1c3cf0a0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               type      src  \\\n",
              "0              uses  betal18   \n",
              "1      consequences  setal08   \n",
              "2              uses    dod20   \n",
              "3         instances  setal08   \n",
              "4              uses    dod20   \n",
              "...             ...      ...   \n",
              "41540     instances   motesp   \n",
              "41541  consequences  setal08   \n",
              "41542          uses     hmsl   \n",
              "41543          uses   motesf   \n",
              "41544          uses  betal18   \n",
              "\n",
              "                                                question    prompt  \\\n",
              "0                   What is a surprising use for a ROPE?      rope   \n",
              "1      What would be a surprising consequence if PEOP...  no sleep   \n",
              "2                  What is a surprising use for a BRICK?     brick   \n",
              "3              What is a surprising thing that is ROUND?     round   \n",
              "4                   What is a surprising use for a BOOK?      book   \n",
              "...                                                  ...       ...   \n",
              "41540    What is a surprising example of something COLD?      cold   \n",
              "41541  What would be a surprising consequence if PEOP...  no sleep   \n",
              "41542              What is a surprising use for a BRICK?     brick   \n",
              "41543              What is a surprising use for a SPOON?     spoon   \n",
              "41544               What is a surprising use for a ROPE?      rope   \n",
              "\n",
              "                                                response  \\\n",
              "0                                              handcuffs   \n",
              "1                                                    fat   \n",
              "2                                                   mold   \n",
              "3                                                  earth   \n",
              "4                                          entertainment   \n",
              "...                                                  ...   \n",
              "41540  In a story book, a villain's heart is colder t...   \n",
              "41541                there would be no use for nighttime   \n",
              "41542                                   build a doghouse   \n",
              "41543             to melt it to turn it into an elephant   \n",
              "41544                                      dream catcher   \n",
              "\n",
              "                          id  target   participant response_num  count  \n",
              "0        betal18_rope-0c1062     1.2   betal182049         None    9.0  \n",
              "1         setal08_3.0-4a78a7     1.3     setal0841          4.0    NaN  \n",
              "2         dod20_brick-cd15cb     2.2        dod208           15    2.0  \n",
              "3         setal08_2.0-3a4509     1.3    setal08136         14.0   14.0  \n",
              "4          dod20_book-562c60     1.5       dod2054            3    5.0  \n",
              "...                      ...     ...           ...          ...    ...  \n",
              "41540  motesp_g2_cold-a7a7c2     4.6    motesp19HO         None    NaN  \n",
              "41541     setal08_3.0-4f3a92     1.0    setal08156          4.0    NaN  \n",
              "41542      hmsl_brick-e68816     1.7  hmsl2E2qy4K0          7.0    4.0  \n",
              "41543    motesf_spoon-7fb131     4.4  motesfc74d97            2    NaN  \n",
              "41544    betal18_rope-a5cec2     2.7   betal182126         None    2.0  \n",
              "\n",
              "[41545 rows x 10 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7d21d32f-451d-4a77-a9f6-a0f3cdef8531\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>uses</td>\n",
              "      <td>betal18</td>\n",
              "      <td>What is a surprising use for a ROPE?</td>\n",
              "      <td>rope</td>\n",
              "      <td>handcuffs</td>\n",
              "      <td>betal18_rope-0c1062</td>\n",
              "      <td>1.2</td>\n",
              "      <td>betal182049</td>\n",
              "      <td>None</td>\n",
              "      <td>9.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>consequences</td>\n",
              "      <td>setal08</td>\n",
              "      <td>What would be a surprising consequence if PEOP...</td>\n",
              "      <td>no sleep</td>\n",
              "      <td>fat</td>\n",
              "      <td>setal08_3.0-4a78a7</td>\n",
              "      <td>1.3</td>\n",
              "      <td>setal0841</td>\n",
              "      <td>4.0</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>uses</td>\n",
              "      <td>dod20</td>\n",
              "      <td>What is a surprising use for a BRICK?</td>\n",
              "      <td>brick</td>\n",
              "      <td>mold</td>\n",
              "      <td>dod20_brick-cd15cb</td>\n",
              "      <td>2.2</td>\n",
              "      <td>dod208</td>\n",
              "      <td>15</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>instances</td>\n",
              "      <td>setal08</td>\n",
              "      <td>What is a surprising thing that is ROUND?</td>\n",
              "      <td>round</td>\n",
              "      <td>earth</td>\n",
              "      <td>setal08_2.0-3a4509</td>\n",
              "      <td>1.3</td>\n",
              "      <td>setal08136</td>\n",
              "      <td>14.0</td>\n",
              "      <td>14.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>uses</td>\n",
              "      <td>dod20</td>\n",
              "      <td>What is a surprising use for a BOOK?</td>\n",
              "      <td>book</td>\n",
              "      <td>entertainment</td>\n",
              "      <td>dod20_book-562c60</td>\n",
              "      <td>1.5</td>\n",
              "      <td>dod2054</td>\n",
              "      <td>3</td>\n",
              "      <td>5.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41540</th>\n",
              "      <td>instances</td>\n",
              "      <td>motesp</td>\n",
              "      <td>What is a surprising example of something COLD?</td>\n",
              "      <td>cold</td>\n",
              "      <td>In a story book, a villain's heart is colder t...</td>\n",
              "      <td>motesp_g2_cold-a7a7c2</td>\n",
              "      <td>4.6</td>\n",
              "      <td>motesp19HO</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41541</th>\n",
              "      <td>consequences</td>\n",
              "      <td>setal08</td>\n",
              "      <td>What would be a surprising consequence if PEOP...</td>\n",
              "      <td>no sleep</td>\n",
              "      <td>there would be no use for nighttime</td>\n",
              "      <td>setal08_3.0-4f3a92</td>\n",
              "      <td>1.0</td>\n",
              "      <td>setal08156</td>\n",
              "      <td>4.0</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41542</th>\n",
              "      <td>uses</td>\n",
              "      <td>hmsl</td>\n",
              "      <td>What is a surprising use for a BRICK?</td>\n",
              "      <td>brick</td>\n",
              "      <td>build a doghouse</td>\n",
              "      <td>hmsl_brick-e68816</td>\n",
              "      <td>1.7</td>\n",
              "      <td>hmsl2E2qy4K0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>4.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41543</th>\n",
              "      <td>uses</td>\n",
              "      <td>motesf</td>\n",
              "      <td>What is a surprising use for a SPOON?</td>\n",
              "      <td>spoon</td>\n",
              "      <td>to melt it to turn it into an elephant</td>\n",
              "      <td>motesf_spoon-7fb131</td>\n",
              "      <td>4.4</td>\n",
              "      <td>motesfc74d97</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41544</th>\n",
              "      <td>uses</td>\n",
              "      <td>betal18</td>\n",
              "      <td>What is a surprising use for a ROPE?</td>\n",
              "      <td>rope</td>\n",
              "      <td>dream catcher</td>\n",
              "      <td>betal18_rope-a5cec2</td>\n",
              "      <td>2.7</td>\n",
              "      <td>betal182126</td>\n",
              "      <td>None</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>41545 rows × 10 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7d21d32f-451d-4a77-a9f6-a0f3cdef8531')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7d21d32f-451d-4a77-a9f6-a0f3cdef8531 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7d21d32f-451d-4a77-a9f6-a0f3cdef8531');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "f7ps_G3makbo",
        "outputId": "12aad5ff-6673-489c-cb0b-971d9427614f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pre-dedupe data size is 27206 items\n",
            "# of unique participants is 2040\n",
            "# of unique prompts 21\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "dod20      5490\n",
              "snbmo09    4099\n",
              "hmsl       3843\n",
              "setal08    3425\n",
              "betal18    2918\n",
              "motesf     2913\n",
              "snb17      2372\n",
              "bs12       1807\n",
              "motesp      339\n",
              "Name: src, dtype: int64"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "proportion of representation by each prompt\n",
            "brick         27.83\n",
            "box           15.99\n",
            "rope          11.46\n",
            "knife         10.55\n",
            "paperclip      6.47\n",
            "bottle         3.55\n",
            "book           2.56\n",
            "table          2.13\n",
            "pants          1.97\n",
            "tire           1.91\n",
            "fork           1.90\n",
            "shoe           1.79\n",
            "shovel         1.55\n",
            "spoon          1.47\n",
            "sock           1.47\n",
            "ball           1.47\n",
            "toothbrush     1.46\n",
            "lightbulb      1.46\n",
            "hat            1.45\n",
            "pencil         1.44\n",
            "backpack       0.12\n",
            "Name: prompt, dtype: float64\n",
            "\n",
            "Number of datasets with each prompt\n",
            "brick         5\n",
            "rope          3\n",
            "bottle        3\n",
            "box           3\n",
            "sock          2\n",
            "knife         2\n",
            "lightbulb     2\n",
            "pencil        2\n",
            "hat           2\n",
            "shoe          2\n",
            "ball          2\n",
            "toothbrush    2\n",
            "spoon         2\n",
            "shovel        1\n",
            "pants         1\n",
            "paperclip     1\n",
            "tire          1\n",
            "table         1\n",
            "fork          1\n",
            "book          1\n",
            "backpack      1\n",
            "Name: prompt, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "data = pd.concat(datasets.values()).sample(frac=1, random_state=1234)\n",
        "\n",
        "# Fix prompt grammar and spelling\n",
        "data.question = data.question.str.replace('a (PANTS)', r'\\1', regex=True).str.replace('LIGHTBULB', 'LIGHT BULB')\n",
        "#data.prompt = data.prompt.str.replace('lightbulb', 'light bulb')\n",
        "\n",
        "#@markdown save all data before deduplication for easy access later?\n",
        "save_all = True #param {type:'boolean'}\n",
        "save_location = base_dir / 'all_data.csv' #@param {type:'raw'}\n",
        "if save_all:\n",
        "    data.to_csv(save_location)\n",
        "\n",
        "#@markdown For generating splits - focus just on aut?\n",
        "aut_only = False #@param {type:'boolean'}\n",
        "if aut_only:\n",
        "    data = data[data.type == 'uses']\n",
        "\n",
        "drop_missing = True #@param {type:'boolean'}\n",
        "if drop_missing:\n",
        "    data = data[~data.target.isna()]\n",
        "print(f\"Pre-dedupe data size is {len(data)} items\")\n",
        "print(f'# of unique participants is {len(data.participant.unique())}')\n",
        "print(\"# of unique prompts\", len(data.prompt.unique()))\n",
        "\n",
        "display(data['src'].value_counts())\n",
        "\n",
        "combine_gt_for_dupes = True #@param {type:'boolean'}\n",
        "if combine_gt_for_dupes:\n",
        "    dupe_means = data.groupby(['prompt', 'response'], as_index=False).target.aggregate(['count', 'mean', 'std']).reset_index().round(1)\n",
        "    dupe_means = dupe_means[dupe_means['count'] > 1]\n",
        "    data = data.merge(dupe_means, how='left')\n",
        "    data.loc[~data['mean'].isna(), 'target'] = data['mean'].dropna() # set target to be mean\n",
        "    data = data.drop(columns=['mean', 'std'])\n",
        "\n",
        "ensure_no_dupes = True #@param {type: 'boolean'}\n",
        "deduplication_strategy = \"Uncased Match\" #@param [\"Exact\", \"Uncased Match\"]\n",
        "og_size = len(data)\n",
        "if ensure_no_dupes:\n",
        "    if deduplication_strategy == \"Uncased Match\":\n",
        "        data['response_lower'] = data.response.str.lower()\n",
        "        data = data.drop_duplicates(['prompt', 'response_lower'])\n",
        "        data = data.drop('response_lower', axis=1)\n",
        "    elif deduplication_strategy == \"Exact\":\n",
        "        data = data.drop_duplicates(['prompt', 'response'])\n",
        "    print(f\"Dropped {og_size-len(data)} duplicate items. {100*(og_size-len(data))/og_size:.2f}%\")\n",
        "    print(f\"Final data size is {len(data)} items\")\n",
        "\n",
        "print('\\nproportion of representation by each prompt')\n",
        "print((data.prompt.value_counts() / len(data)).multiply(100).round(2))\n",
        "\n",
        "print(\"\\nNumber of datasets with each prompt\")\n",
        "print(data[['prompt', 'src']].drop_duplicates().prompt.value_counts())"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scipy.stats import entropy\n",
        "data.groupby('prompt').target.apply(entropy).sort_values().round(3).to_dict()"
      ],
      "metadata": {
        "id": "YzS9PSnGzo9f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GN4OT7sLKsK_"
      },
      "source": [
        "## Save Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H65oin4lXOI8",
        "outputId": "fc48541c-5edb-4190-bb35-3345b308b2b4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "####gt_main_with_dupes\n",
            "- Data size is 27206 items\n",
            "seed 987\n",
            "targetsplits {'train': 80, 'val': 5, 'test': 15}\n",
            "split_by_part: False; split_by_prompt: False\n",
            "Final split sizes: [80.0, 5.0, 15.0]\n"
          ]
        }
      ],
      "source": [
        "name = 'gt_main_with_dupes' #@param {type:'string'}\n",
        "random_seed = 987 #@param {type:'number'}\n",
        "\n",
        "# ensure subfolder to avoid rm'ing something dumb\n",
        "data_dir = os.path.join('data', name)\n",
        "!rm -rf $data_dir\n",
        "\n",
        "print(f'####{name}')\n",
        "print(f\"- Data size is {len(data)} items\")\n",
        "os.makedirs(data_dir, exist_ok=True)\n",
        "splits = {'train': 80, 'val': 5, 'test': 15} #@param {type: 'raw'}\n",
        "print('seed', random_seed)\n",
        "print('targetsplits', splits)\n",
        "#markdown ### Data to Include\n",
        "\n",
        "#@markdown ### Parameters\n",
        "#@markdown If splitting data by participant, individuals will be entirely in a single split.\n",
        "#@markdown If paired with ensure_no_dupes, the test is split out, then the train data is deduped\n",
        "#@markdown so no test data is seen. This will imbalance the split sizes from expectation.\n",
        "split_by_participant = False #@param {type: 'boolean'}\n",
        "#@markdown Split where test data has different prompts than train.\n",
        "split_by_prompt = False #@param {type: 'boolean'}\n",
        "try:\n",
        "    assert not (split_by_participant and split_by_prompt)\n",
        "except:\n",
        "    raise AssertionError(\"Can't split by participant *and* prompt\")\n",
        "\n",
        "print(f'split_by_part: {split_by_participant}; split_by_prompt: {split_by_prompt}')\n",
        "\n",
        "#@markdown `dry_run`: don't save outputs\n",
        "dry_run = False #@param {type: 'boolean'}\n",
        "\n",
        "examples = dict()\n",
        "\n",
        "def fingerprint(x):\n",
        "    x = x.lower()\n",
        "    x = \"\".join([c for c in x if c.isalpha()])\n",
        "    return x\n",
        "\n",
        "if split_by_participant or split_by_prompt:\n",
        "    try:\n",
        "        assert set(examples.keys()) == {'test', 'train', 'val'}\n",
        "    except:\n",
        "        raise Exception('split_by_participant or split_by_prompt requires test/train/val names')\n",
        "\n",
        "    sample = dict()\n",
        "    splitcol = 'participant' if split_by_participant else 'prompt'\n",
        "    if split_by_participant:\n",
        "        unique_vals = data[splitcol].unique().tolist()\n",
        "\n",
        "        sample['train'], sample['test'] = train_test_split(unique_vals, train_size=splits['train']/sum(splits.values()), shuffle=True, random_state=random_seed)\n",
        "        sample['val'], sample['test']  = train_test_split(sample['test'], train_size=splits['val']/(splits['val']+splits['test']), shuffle=True, random_state=random_seed)\n",
        "    elif split_by_prompt:\n",
        "        # most common prompts\n",
        "        promptn = data.prompt.value_counts() / len(data)\n",
        "        # put aside the n most plentiful prompts, and add them to train later\n",
        "        sample['train'] = promptn[:4].index.tolist()\n",
        "        # recalc, to use splits based on the rest of the data\n",
        "        #remaining = data[~data.prompt.isin(sample['train'])]\n",
        "        #promptn = remaining.prompt.value_counts() / len(remaining)\n",
        "        promptn = promptn[~promptn.index.isin(sample['train'])]\n",
        "\n",
        "        # fill val and test, then add remainder to train\n",
        "        pool = promptn.sample(frac=1, random_state=random_seed).cumsum()\n",
        "        sample['val'] = pool[(pool-promptn.mean()/2) < splits['val']/100].index.tolist()\n",
        "        promptn = promptn[~promptn.index.isin(sample['val'])]\n",
        "\n",
        "        pool = promptn.sample(frac=1, random_state=random_seed).cumsum()\n",
        "        sample['test'] = pool[(pool-promptn.mean()/2) < splits['test']/100].index.tolist()\n",
        "        promptn = promptn[~promptn.index.isin(sample['test'])]\n",
        "\n",
        "        sample['train'] += promptn.index.tolist()\n",
        "        print('train:', sample['train'])\n",
        "        print('val:', sample['val'])\n",
        "        print('test:', sample['test'])\n",
        "\n",
        "    for splittype in splits.keys():\n",
        "        examples[splittype] = data[data[splitcol].isin(sample[splittype])].to_dict(orient='records')\n",
        "\n",
        "    if ensure_no_dupes and split_by_participant: #split by prompt is already deduped\n",
        "        exclude_list = [fingerprint(x['prompt']+x['response']) for x in examples['test']+examples['val']]\n",
        "        examples['train_og'] = examples['train']\n",
        "        examples['train'] = [x for x in examples['train'] if fingerprint(x['prompt']+x['response']) not in exclude_list]\n",
        "\n",
        "else:\n",
        "    # this approach allows custom split names\n",
        "    data_shuff = data.sample(frac=1, random_state=random_seed)\n",
        "    n = 0\n",
        "    nd = len(data_shuff)\n",
        "    for k, v in splits.items():\n",
        "        m = np.rint(n + (nd * v)/100).astype(int)\n",
        "        examples[k] = data_shuff.iloc[n:m].to_dict(orient='records')\n",
        "        n = m\n",
        "\n",
        "sizes = [len(examples[splittype]) for splittype in splits.keys()]\n",
        "print(\"Final split sizes:\", [np.round(100*x/sum(sizes), 1) for x in sizes])\n",
        "\n",
        "if not dry_run:\n",
        "    for split_type, split in splits.items():\n",
        "        os.makedirs(os.path.join(data_dir, split_type), exist_ok=True)\n",
        "\n",
        "        for item in examples[split_type]:\n",
        "            with open(os.path.join(data_dir, split_type, f\"{item['id']}.json\"), mode='w') as f:\n",
        "                json.dump(item, f)\n",
        "\n",
        "    !tar -czf \"{name}.tar.gz\" data/${name}\n",
        "    !mv \"{name}.tar.gz\" \"{base_dir}\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KHocMN_pQABA"
      },
      "source": [
        "# Dataset Reference\n",
        "\n",
        "####gt_main2\n",
        "\n",
        "This is the main split in the first LLM paper.\n",
        "\n",
        "- Data size is 20202 items\n",
        "- seed 987\n",
        "- targetsplits {'train': 80, 'val': 5, 'test': 15}\n",
        "- split_by_part: False; split_by_prompt: False\n",
        "- Final split sizes: [80.0, 5.0, 15.0]\n",
        "- (gt_main_std *should* be identical, with stdev included, but I haven't\n",
        "doublechecked)\n",
        "\n",
        "#### gt_main_with_dupes\n",
        "\n",
        "- same context as above, but not deduplicated. This is a data leakage context, which should be avoided, though to compare with past work we trained one such model.\n",
        "\n",
        "####gt_byparticipant\n",
        "- Data size is 20202 items\n",
        "- seed 987\n",
        "- targetsplits {'train': 80, 'val': 5, 'test': 15}\n",
        "- split_by_part: True; split_by_prompt: False\n",
        "- Final split sizes: [80.7, 4.7, 14.6]\n",
        "\n",
        "####gt_byprompt\n",
        "\n",
        "This the the split used for having different prompts between test and train in the first LLM paper.\n",
        "\n",
        "- Data size is 20202 items\n",
        "- seed 987\n",
        "- targetsplits {'train': 79, 'val': 4, 'test': 17}\n",
        "- split_by_part: False; split_by_prompt: True\n",
        "- train: ['brick', 'box', 'knife', 'rope', 'book', 'table', 'tire', 'fork', 'ball', 'pencil', 'lightbulb', 'shoe', 'hat', 'sock', 'toothbrush', 'backpack']\n",
        "- val: []\n",
        "- test: ['paperclip', 'spoon', 'bottle', 'shovel', 'pants']\n",
        "- Final split sizes: [83.2, 0.0, 16.8]\n",
        "\n",
        "####all\n",
        "\n",
        "This is the condition for training the final model, for use in the Open Creativity Scoring system. It was *not* deduped for training to avoid data leakeage, since it's for applied use and that performance would be desired.\n",
        "\n",
        "- Data size is 27217 items\n",
        "- seed 987\n",
        "- targetsplits {'train': 94, 'val': 1, 'test': 5}\n",
        "- split_by_part: False; split_by_prompt: False\n",
        "- Final split sizes: [94.0, 1.0, 5.0]\n",
        "\n",
        "####gt_alltests2\n",
        "\n",
        "This is the condition which includes consequences, instances, and complete the sentence. It may grow outdated as I develop a format for training with this data.\n",
        "\n",
        "It doesn't have test/train, rather multiple numbered groups so that an ensemble can be trained.\n",
        "\n",
        "- Data size is 31567 items\n",
        "- seed 987\n",
        "- targetsplits {'group1': 32, 'group2': 32, 'group3': 32, 'val': 4}\n",
        "- split_by_part: False; split_by_prompt: False\n",
        "- Final split sizes: [32.0, 32.0, 32.0, 4.0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uJon5X15GaV4",
        "outputId": "d9d0e19e-ead5-4f01-e524-ca2cb4eec923"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "All gt options\n",
            "['gt_main', 'gt_bypart3', 'gt_byprompt4', 'gt_byparticipant', 'gt_byprompt', 'all', 'gt_main2', 'gt_main_std', 'gt_alltests1']\n"
          ]
        }
      ],
      "source": [
        "dprint(\"All gt options\")\n",
        "print([x.stem.split('.')[0] for x in base_dir.glob('*tar.gz')])"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "zltQcwkXeRvk",
        "gIwAL3uqaikc",
        "jn8ziOcamzaH",
        "vJqyd2UHDCMZ",
        "UH3yOwIon4Ky",
        "VgNoUTqg_dQb",
        "laa40OuiNV2X",
        "8H-WPj9MeGrS",
        "tbICpoGS4nvR"
      ],
      "provenance": [],
      "mount_file_id": "1bZ0PDifSqhWhAUwuLK4BCt0Mke50ww-Q",
      "authorship_tag": "ABX9TyOiLWVlkci39ayG4qPrCLCX",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}