#!/usr/bin/env Rscript # Copyright 2026 Mike Hammes. SPDX-License-Identifier: Apache-2.0 # Independent CSV-based reference, explicit Welch formulas, then NLSS comparison. # Usage: Rscript scripts/check-reference.R PROJECT OUTPUT_JSON # OUTPUT_JSON must not exist. Dependencies: jsonlite, digest, arrow. args <- commandArgs(trailingOnly = TRUE) if (length(args) != 2L) stop("Supply project directory and new JSON output path") root <- normalizePath(args[[1L]], mustWork = TRUE) if (file.exists(args[[2L]])) stop("Refusing to replace a verification result") expected_hash <- "3dab41d156991ad77511263ebea54ed33946da2add84dfed95d9c8f6bca0fe72" csv <- file.path(root, "learning-study.csv") d <- read.csv(csv, stringsAsFactors = FALSE, check.names = FALSE) checks <- list() check <- function(name, ok, actual = NULL, expected = NULL) { checks[[length(checks) + 1L]] <<- list(name = name, pass = isTRUE(ok), actual = actual, expected = expected) } near <- function(name, actual, expected) { ok <- is.numeric(actual) && is.numeric(expected) && length(actual) == length(expected) && length(actual) > 0L && all(is.finite(actual)) && all(is.finite(expected)) && all(abs(actual - expected) <= 1e-8 + 1e-7 * abs(expected)) check(name, ok, actual, expected) } exact <- function(name, actual, expected) check(name, identical(actual, expected), actual, expected) exact("frozen_csv_sha256", digest::digest(file = csv, algo = "sha256"), expected_hash) exact("column_names", names(d), c("id", "condition", "score")) exact("rows", nrow(d), 80L) exact("unique_ids", sort(d$id), 1:80) exact("group_codes", unique(d$condition), c("practice", "rereading")) check("no_missing_values", !anyNA(d)) check("integer_score_0_100", all(d$score %in% 0:100)) check("export_order", identical(order(match(d$condition, c("practice", "rereading")), d$id), 1:80)) x <- d$score[d$condition == "practice"] y <- d$score[d$condition == "rereading"] exact("balanced_groups", c(length(x), length(y)), c(40L, 40L)) nx <- length(x); ny <- length(y) mx <- mean(x); my <- mean(y); sx <- sd(x); sy <- sd(y) delta <- mx - my se <- sqrt(sx^2/nx + sy^2/ny) df <- (sx^2/nx + sy^2/ny)^2 / ((sx^2/nx)^2/(nx-1) + (sy^2/ny)^2/(ny-1)) t_value <- delta/se p_value <- 2 * pt(-abs(t_value), df) ci <- delta + c(-1, 1) * qt(0.975, df) * se sp <- sqrt(((nx-1)*sx^2 + (ny-1)*sy^2)/(nx+ny-2)) cohen_d <- delta/sp base_fit <- stats::t.test(x, y, var.equal = FALSE, alternative = "two.sided", conf.level = 0.95) reference <- list(n_1 = nx, n_2 = ny, mean_1 = mx, mean_2 = my, sd_1 = sx, sd_2 = sy, mean_diff = delta, stderr = se, t = t_value, df = df, p = p_value, ci_low = ci[[1]], ci_high = ci[[2]], d = cohen_d) near("base_R_vs_formula_t", unname(base_fit$statistic), t_value) near("base_R_vs_formula_df", unname(base_fit$parameter), df) near("base_R_vs_formula_p", base_fit$p.value, p_value) near("base_R_vs_formula_CI", as.numeric(base_fit$conf.int), ci) # No NLSS results have been read above this line. readj <- function(p) jsonlite::fromJSON(p, simplifyVector = FALSE) paths <- list.files(file.path(root, ".nlss/runs"), "^result.json$", recursive = TRUE, full.names = TRUE) runs <- lapply(paths, readj) pick <- function(module) { hit <- which(vapply(runs, function(r) identical(r$module, module), logical(1))) if (length(hit) != 1L) stop("Expected one reference run for ", module) j <- runs[[hit]] if (!identical(j$status, "completed")) stop("Incomplete run: ", module) list(result = j, request = readj(file.path(dirname(paths[[hit]]), "request.json"))) } tt <- pick("t_test"); ds <- pick("descriptive_stats"); pl <- pick("plot") aa <- pick("assumptions"); ex <- pick("data_explorer") tr <- tt$result$results$summary_df[[1]] for (name in names(reference)) near(paste0("NLSS_", name), tr[[name]], reference[[name]]) exact("test_group_1", tr$group_1, "practice") exact("test_group_2", tr$group_2, "rereading") exact("test_n_exact", c(as.integer(tr$n_1), as.integer(tr$n_2)), c(nx, ny)) exact("CI_estimand", tr$ci_estimand, "population_mean_difference") exact("d_definition", tr$d_definition, "pooled_sample_sd") exact("bootstrap_d_not_requested", tr$boot_d_ci_status, "not_requested") exact("Welch_var_equal_false", tt$request$options$var_equal, FALSE) exact("two_sided", tt$request$options$alternative, "two.sided") near("confidence_level", tt$request$options$conf_level, 0.95) exact("bootstrap_false", tt$request$options$bootstrap, FALSE) exact("request_group_order", vapply(tt$request$design$group_order, `[[`, character(1), "value"), c("practice", "rereading")) exact("analysed_rows", as.integer(unlist(tt$request$design$tests[[1]]$row_indices)), 1:80) exact("excluded_rows", length(tt$request$design$tests[[1]]$excluded_row_indices), 0L) group_reference <- list() for (g in c("practice", "rereading")) { v <- d$score[d$condition == g] sr <- Filter(function(r) r$group == g, ds$result$results$summary_df)[[1]] gs <- list(n = length(v), missing_n = 0L, mean = mean(v), sd = sd(v), min = min(v), max = max(v), median = median(v), q1 = unname(quantile(v, .25)), q3 = unname(quantile(v, .75))) for (name in names(gs)) near(paste("descriptives", g, name, sep = "_"), sr[[name]], gs[[name]]) group_reference[[g]] <- gs } raw_snapshot <- as.data.frame(arrow::read_parquet(file.path(root, tt$request$dataset$snapshot_path))) exact("analysed_input_column_names", names(raw_snapshot), names(d)) exact("analysed_input_row_count", nrow(raw_snapshot), nrow(d)) for (name in names(d)) exact(paste0("analysed_input_column_", name), raw_snapshot[[name]], d[[name]]) # NLSS adds a frame-level import-contract attribute; it is not CSV cell data. contract <- attr(raw_snapshot, "nlss_import_contract") exact("snapshot_contract_source_rows", contract$source_rows, nrow(d)) for (r in list(tt, ds, pl, aa, ex)) { exact(paste0(r$result$module, "_source_hash"), r$result$dataset$source_sha256, expected_hash) exact(paste0(r$result$module, "_same_input"), r$result$dataset$data_sha256, tt$result$dataset$data_sha256) } plot <- pl$result$results$plots[[1]] exact("plot_rows", as.integer(unlist(plot$cases$source_rows)), 1:80) exact("plot_excluded", length(plot$cases$excluded_rows), 0L) exact("plot_scores", as.integer(vapply(plot$data, `[[`, numeric(1), ".y")), d$score) exact("plot_groups", vapply(plot$data, `[[`, character(1), ".x"), d$condition) for (i in 1:2) { v <- if (i == 1L) x else y q <- as.numeric(quantile(v, c(.25, .5, .75))) bounds <- c(q[1] - 1.5*(q[3]-q[1]), q[3] + 1.5*(q[3]-q[1])) inside <- v[v >= bounds[1] & v <= bounds[2]] lyr <- plot$layers[[1]][[i]] near(paste0("box_", i, "_quartiles"), c(lyr$lower, lyr$middle, lyr$upper), q) near(paste0("box_", i, "_whiskers"), c(lyr$ymin, lyr$ymax), range(inside)) exact(paste0("box_", i, "_outlier_count"), length(lyr$outliers), length(v)-length(inside)) } diag <- aa$result$results$checks_df for (g in c("practice", "rereading")) { sr <- Filter(function(r) r$test == "Shapiro-Wilk" && r$group == g, diag)[[1]] sw <- stats::shapiro.test(d$score[d$condition == g]) near(paste0("shapiro_",g,"_W"), sr$statistic, unname(sw$statistic)) near(paste0("shapiro_",g,"_p"), sr$p, sw$p.value) } # Levene's median-centred absolute deviations, independent formula via base lm. z <- abs(d$score - ave(d$score, d$condition, FUN = median)) lev <- anova(lm(z ~ d$condition)) sr <- Filter(function(r) r$test == "Levene (median)", diag)[[1]] near("Levene_F", sr$statistic, lev[["F value"]][1]) near("Levene_p", sr$p, lev[["Pr(>F)"]][1]) passed <- all(vapply(checks, `[[`, logical(1), "pass")) out <- list(material_version = "1.0", source_sha256 = expected_hash, method = "CSV-only stats::t.test and explicit Welch/pooled-SD formulas; then compare NLSS evidence", tolerance = "abs(actual-reference) <= 1e-8 + 1e-7*abs(reference); exact cases and coding", caveat = "Independent workflow/formulas, not an independent statistical software engine; base R distribution functions shared.", direction = "practice minus rereading", reference = reference, groups = group_reference, reference_run_ids = vapply(list(ex, ds, aa, tt, pl), function(r) r$result$run_id, character(1)), r_version = R.version.string, all_pass = passed, check_count = length(checks), checks = checks) jsonlite::write_json(out, args[[2L]], pretty = TRUE, auto_unbox = TRUE, digits = NA, null = "null") cat(jsonlite::toJSON(list(reference=reference,checks=length(checks),all_pass=passed),auto_unbox=TRUE,digits=NA,pretty=TRUE), "\n") if (!passed) quit(status = 1L)