Language models accurately infer correlations between psychological items and scales from text alone

Here, we apply the model to data not used for training, a holdout. The holdout sample was collected by Bainbridge et al. 2022.

knitr::opts_chunk$set(echo = TRUE, error = T)

# Libraries and Settings

# Libs ---------------------------
library(tidyverse)
library(ggplot2)
library(plotly)

item_pair_table <- rio::import("ignore/item_pair_table.feather")
set.seed(42)
(item_pair_table %>% 
  mutate(synthetic_r = round(synthetic_r, 2),
         empirical_r = round(empirical_r, 2),
         items = str_replace_all(str_c(item_text_1, "\n", item_text_2),
                                  "_+", " ")) %>% 
    # sample_df(2000) %>% 
ggplot(., aes(synthetic_r, empirical_r, 
              # ymin = empirical_r - empirical_r_se, 
              # ymax = empirical_r + empirical_r_se, 
              label = items)) + 
  geom_abline(linetype = "dashed") +
  geom_point(color = "#00A0B0", alpha = 0.3, size = 1) +
  xlab("Synthetic inter-item correlation") + 
  ylab("Empirical inter-item correlation") +
  theme_bw() +
  coord_fixed(xlim = c(-1,1), ylim = c(-1,1))) %>% 
  ggplotly()