Language models accurately infer correlations between psychological items and scales from text alone

Here, we apply the model to the data collected in our Registered Report validation sample on Nov 1, 2024.

knitr::opts_chunk$set(echo = TRUE, error = T)

# Libraries and Settings

# Libs ---------------------------
library(tidyverse)
library(ggplot2)
library(plotly)

item_pair_table <- rio::import("ignore/item_pair_table_rr.feather")
set.seed(42)
(item_pair_table %>% 
  mutate(synthetic_r = round(synthetic_r, 2),
         empirical_r = round(empirical_r, 2),
         items = str_replace_all(str_c(item_text_1, "\n", item_text_2),
                                  "_+", " ")) %>% 
    # sample_df(2000) %>% 
ggplot(., aes(synthetic_r, empirical_r, 
              # ymin = empirical_r - empirical_r_se, 
              # ymax = empirical_r + empirical_r_se, 
              label = items)) + 
  geom_abline(linetype = "dashed") +
  geom_point(color = "#00A0B0", alpha = 0.3, size = 1) +
  xlab("Synthetic inter-item correlation") + 
  ylab("Empirical inter-item correlation") +
  theme_bw() +
  coord_fixed(xlim = c(-1,1), ylim = c(-1,1))) %>% 
  ggplotly()