Add files via upload

Makefile

@@ -0,0 +1,24 @@
+CC = gcc
+CFLAGS = -O3 -march=native -Wall -Wextra
+LDFLAGS = -lm
+
+HEADERS = arith.h ppm.h tweedie.h match.h word.h highctx.h fastmath.h
+
+all: mdc ablation
+
+mdc: mdc.c $(HEADERS)
+	$(CC) $(CFLAGS) -o mdc mdc.c $(LDFLAGS)
+
+ablation: ablation.c $(HEADERS)
+	$(CC) $(CFLAGS) -o ablation ablation.c $(LDFLAGS)
+
+test_arith: test_arith.c arith.h
+	$(CC) $(CFLAGS) -o test_arith test_arith.c $(LDFLAGS)
+
+test_ppm: test_ppm.c arith.h ppm.h
+	$(CC) $(CFLAGS) -o test_ppm test_ppm.c $(LDFLAGS)
+
+clean:
+	rm -f mdc ablation bench test_arith test_ppm
+
+.PHONY: all clean

ablation.c

@@ -0,0 +1,377 @@
+/*
+ * Ablation study — C implementation
+ * Measures incremental contribution of each pipeline layer.
+ *
+ * Usage:
+ *   ./ablation                     # alice29.txt only
+ *   ./ablation file1 file2 ...     # specific files
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+#include <libgen.h>
+
+#include "fastmath.h"
+#include "arith.h"
+#include "ppm.h"
+#include "tweedie.h"
+#include "match.h"
+#include "word.h"
+#include "highctx.h"
+
+#define SCALE (1 << 14)
+
+/* ── Flags ── */
+#define FLAG_TWEEDIE    1
+#define FLAG_MATCH      2
+#define FLAG_WORD       4
+#define FLAG_HIGHCTX    8
+
+/* ── Helpers ── */
+
+static void probs_to_cumfreqs(const double *probs, int64_t *cumfreqs,
+                               int64_t *out_total) {
+    cumfreqs[0] = 0;
+    for (int i = 0; i < 256; i++) {
+        int64_t f = (int64_t)(probs[i] * SCALE + 0.5);
+        if (f < 1) f = 1;
+        cumfreqs[i + 1] = cumfreqs[i] + f;
+    }
+    *out_total = cumfreqs[256];
+}
+
+static void clamp_normalize(double *probs) {
+    double sum = 0.0;
+    for (int i = 0; i < 256; i++) {
+        if (probs[i] < 1e-10) probs[i] = 1e-10;
+        sum += probs[i];
+    }
+    double inv = 1.0 / sum;
+    for (int i = 0; i < 256; i++)
+        probs[i] *= inv;
+}
+
+static inline double now_sec(void) {
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return ts.tv_sec + ts.tv_nsec * 1e-9;
+}
+
+/* ── Configurable compress ── */
+
+static uint8_t *do_compress(const uint8_t *data, size_t data_len,
+                             int flags, size_t *out_len, double *out_time) {
+    PPMModel ppm;       ppm_init(&ppm);
+    MatchModel match;   if (flags & FLAG_MATCH) match_init(&match);
+    WordModel word;     if (flags & FLAG_WORD) word_init(&word);
+    HighCtxModel hctx;  if (flags & FLAG_HIGHCTX) highctx_init(&hctx);
+    ArithEncoder enc;   ae_init(&enc);
+    TweedieDenoiser twd; if (flags & FLAG_TWEEDIE) tweedie_init(&twd);
+
+    double probs[256], word_probs[256], hctx_probs[256];
+    int64_t cumfreqs[257];
+    int64_t total;
+
+    double t0 = now_sec();
+
+    for (size_t i = 0; i < data_len; i++) {
+        uint8_t byte = data[i];
+        double confidence;
+        int order;
+
+        ppm_predict(&ppm, probs, &confidence, &order);
+
+        if (flags & FLAG_TWEEDIE) {
+            tweedie_denoise(&twd, probs, order, confidence);
+        }
+        clamp_normalize(probs);
+
+        if (flags & FLAG_MATCH) {
+            int match_byte;
+            double match_conf;
+            match_predict(&match, &match_byte, &match_conf);
+            blend_match(probs, match_byte, match_conf);
+        }
+
+        if (flags & FLAG_WORD) {
+            double w_conf;
+            if (word_predict_cached(&word, word_probs, &w_conf))
+                blend_word_model(probs, word_probs, w_conf);
+        }
+
+        if (flags & FLAG_HIGHCTX) {
+            double hctx_conf;
+            if (highctx_predict(&hctx, hctx_probs, &hctx_conf))
+                blend_highctx(probs, hctx_probs, hctx_conf);
+        }
+
+        probs_to_cumfreqs(probs, cumfreqs, &total);
+        ae_encode(&enc, cumfreqs, byte, total);
+
+        /* Updates */
+        if (flags & FLAG_TWEEDIE)
+            tweedie_update(&twd, byte);
+        if (flags & FLAG_MATCH)
+            match_update(&match, byte);
+        if (flags & FLAG_WORD)
+            word_update(&word, byte);
+        if (flags & FLAG_HIGHCTX)
+            highctx_update(&hctx, byte);
+        ppm_update(&ppm, byte);
+
+        if ((i + 1) % 50000 == 0) {
+            double elapsed = now_sec() - t0;
+            double pct = (i + 1) * 100.0 / data_len;
+            double speed = (i + 1) / elapsed;
+            fprintf(stderr, "\r    %5.1f%%  (%zu/%zu)  %.0f B/s",
+                    pct, i + 1, data_len, speed);
+        }
+    }
+
+    ae_finish(&enc);
+    double elapsed = now_sec() - t0;
+    if (data_len >= 50000)
+        fprintf(stderr, "\r                                                \r");
+
+    *out_time = elapsed;
+
+    /* Copy output */
+    *out_len = enc.buf_len;
+    uint8_t *result = (uint8_t *)malloc(enc.buf_len);
+    memcpy(result, enc.buf, enc.buf_len);
+
+    ae_free(&enc);
+    ppm_free(&ppm);
+    if (flags & FLAG_MATCH) match_free(&match);
+    if (flags & FLAG_WORD) word_free(&word);
+    if (flags & FLAG_HIGHCTX) highctx_free(&hctx);
+
+    return result;
+}
+
+/* ── Configurable decompress ── */
+
+static uint8_t *do_decompress(const uint8_t *compressed, size_t comp_len,
+                                size_t original_size, int flags,
+                                double *out_time) {
+    PPMModel ppm;       ppm_init(&ppm);
+    MatchModel match;   if (flags & FLAG_MATCH) match_init(&match);
+    WordModel word;     if (flags & FLAG_WORD) word_init(&word);
+    HighCtxModel hctx;  if (flags & FLAG_HIGHCTX) highctx_init(&hctx);
+    ArithDecoder dec;   ad_init(&dec, compressed, comp_len);
+    TweedieDenoiser twd; if (flags & FLAG_TWEEDIE) tweedie_init(&twd);
+
+    uint8_t *result = (uint8_t *)malloc(original_size);
+
+    double probs[256], word_probs[256], hctx_probs[256];
+    int64_t cumfreqs[257];
+    int64_t total;
+
+    double t0 = now_sec();
+
+    for (size_t i = 0; i < original_size; i++) {
+        double confidence;
+        int order;
+
+        ppm_predict(&ppm, probs, &confidence, &order);
+
+        if (flags & FLAG_TWEEDIE) {
+            tweedie_denoise(&twd, probs, order, confidence);
+        }
+        clamp_normalize(probs);
+
+        if (flags & FLAG_MATCH) {
+            int match_byte;
+            double match_conf;
+            match_predict(&match, &match_byte, &match_conf);
+            blend_match(probs, match_byte, match_conf);
+        }
+
+        if (flags & FLAG_WORD) {
+            double w_conf;
+            if (word_predict_cached(&word, word_probs, &w_conf))
+                blend_word_model(probs, word_probs, w_conf);
+        }
+
+        if (flags & FLAG_HIGHCTX) {
+            double hctx_conf;
+            if (highctx_predict(&hctx, hctx_probs, &hctx_conf))
+                blend_highctx(probs, hctx_probs, hctx_conf);
+        }
+
+        probs_to_cumfreqs(probs, cumfreqs, &total);
+        int sym = ad_decode(&dec, cumfreqs, total);
+        result[i] = (uint8_t)sym;
+
+        if (flags & FLAG_TWEEDIE)
+            tweedie_update(&twd, (uint8_t)sym);
+        if (flags & FLAG_MATCH)
+            match_update(&match, (uint8_t)sym);
+        if (flags & FLAG_WORD)
+            word_update(&word, (uint8_t)sym);
+        if (flags & FLAG_HIGHCTX)
+            highctx_update(&hctx, (uint8_t)sym);
+        ppm_update(&ppm, (uint8_t)sym);
+    }
+
+    *out_time = now_sec() - t0;
+
+    ppm_free(&ppm);
+    if (flags & FLAG_MATCH) match_free(&match);
+    if (flags & FLAG_WORD) word_free(&word);
+    if (flags & FLAG_HIGHCTX) highctx_free(&hctx);
+
+    return result;
+}
+
+/* ── Ablation configs ── */
+
+typedef struct {
+    const char *label;
+    int flags;
+} AblationConfig;
+
+static const AblationConfig CONFIGS[] = {
+    { "Base PPM",                    0 },
+    { "+ Tweedie",                   FLAG_TWEEDIE },
+    { "+ Twd + Match",               FLAG_TWEEDIE | FLAG_MATCH },
+    { "+ Twd + Match + Word",        FLAG_TWEEDIE | FLAG_MATCH | FLAG_WORD },
+    { "+ Twd + M + W + H",           FLAG_TWEEDIE | FLAG_MATCH | FLAG_WORD | FLAG_HIGHCTX },
+};
+#define N_CONFIGS 5
+
+typedef struct {
+    const char *label;
+    size_t c_size;
+    double ratio;
+    double c_time;
+} AblationResult;
+
+static void run_ablation(const char *filepath) {
+    FILE *f = fopen(filepath, "rb");
+    if (!f) { fprintf(stderr, "File not found: %s\n", filepath); return; }
+    fseek(f, 0, SEEK_END);
+    long file_size = ftell(f);
+    fseek(f, 0, SEEK_SET);
+    uint8_t *data = (uint8_t *)malloc(file_size);
+    if (fread(data, 1, file_size, f) != (size_t)file_size) {
+        fprintf(stderr, "Read error: %s\n", filepath);
+        fclose(f); free(data); return;
+    }
+    fclose(f);
+
+    /* basename */
+    char *path_copy = strdup(filepath);
+    const char *filename = basename(path_copy);
+    size_t original_size = (size_t)file_size;
+
+    printf("\n======================================================================\n");
+    printf("  ABLATION: %s (%zu bytes)\n", filename, original_size);
+    printf("======================================================================\n");
+
+    AblationResult results[N_CONFIGS];
+
+    for (int c = 0; c < N_CONFIGS; c++) {
+        printf("\n  [%s]\n", CONFIGS[c].label);
+        printf("    Compressing...");
+        fflush(stdout);
+
+        size_t comp_len;
+        double c_time;
+        uint8_t *compressed = do_compress(data, original_size, CONFIGS[c].flags,
+                                           &comp_len, &c_time);
+        double ratio = (double)comp_len / original_size;
+        printf(" %zu bytes (%.2f%%) in %.1fs\n", comp_len, ratio * 100.0, c_time);
+
+        /* Verify round-trip */
+        printf("    Verifying...");
+        fflush(stdout);
+
+        /*double d_time;
+        uint8_t *decompressed = do_decompress(compressed, comp_len, original_size,
+                                                CONFIGS[c].flags, &d_time);
+
+        if (memcmp(data, decompressed, original_size) == 0) {
+            printf(" OK (%.1fs)\n", d_time);
+        } else {
+            printf(" FAILED!\n");
+            // Find first mismatch
+            for (size_t i = 0; i < original_size; i++) {
+                if (data[i] != decompressed[i]) {
+                    printf("    First mismatch at byte %zu: expected %d, got %d\n",
+                           i, data[i], decompressed[i]);
+                    break;
+                }
+            }
+            free(compressed);
+            free(decompressed);
+            free(data);
+            free(path_copy);
+            exit(1);
+        } */
+
+        results[c].label = CONFIGS[c].label;
+        results[c].c_size = comp_len;
+        results[c].ratio = ratio;
+        results[c].c_time = c_time;
+
+        free(compressed);
+        //free(decompressed);
+    }
+
+    /* ── Summary table ── */
+    printf("\n======================================================================\n");
+    printf("  RESULTS: %s (%zu bytes)\n", filename, original_size);
+    printf("======================================================================\n");
+    printf("  %-32s %8s %8s %9s %9s %7s\n",
+           "Layer", "Size", "Ratio", "Layer +%", "Total +%", "Time");
+    printf("  -------------------------------- -------- -------- --------- --------- -------\n");
+
+    size_t base_size = results[0].c_size;
+    size_t prev_size = results[0].c_size;
+
+    for (int i = 0; i < N_CONFIGS; i++) {
+        size_t c_size = results[i].c_size;
+        double ratio = results[i].ratio;
+        double c_time = results[i].c_time;
+
+        if (i == 0) {
+            printf("  %-32s %8zu %6.2f%% %9s %9s %6.1fs\n",
+                   results[i].label, c_size, ratio * 100.0, "", "", c_time);
+        } else {
+            double layer_imp = (double)(prev_size - c_size) / prev_size * 100.0;
+            double total_imp = (double)(base_size - c_size) / base_size * 100.0;
+            printf("  %-32s %8zu %6.2f%% %+8.2f%% %+8.2f%% %6.1fs\n",
+                   results[i].label, c_size, ratio * 100.0,
+                   layer_imp, total_imp, c_time);
+        }
+        prev_size = c_size;
+    }
+
+    printf("  -------------------------------- -------- -------- --------- --------- -------\n");
+    size_t final_size = results[N_CONFIGS - 1].c_size;
+    double total_imp = (double)(base_size - final_size) / base_size * 100.0;
+    printf("  %-32s %8s %8s %9s %+8.2f%%\n",
+           "TOTAL IMPROVEMENT", "", "", "", total_imp);
+    printf("\n");
+
+    free(data);
+    free(path_copy);
+}
+
+/* ── Main ── */
+
+int main(int argc, char **argv) {
+    if (argc > 1) {
+        for (int i = 1; i < argc; i++)
+            run_ablation(argv[i]);
+    } else {
+        run_ablation("../alice29.txt");
+    }
+
+    /* Cross-file comparison would go here for multiple files */
+    return 0;
+}

arith.h

@@ -0,0 +1,171 @@
+#ifndef ARITH_H
+#define ARITH_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Arithmetic encoder */
+typedef struct {
+    uint32_t low;
+    uint32_t high;
+    int pending;
+    uint8_t *buf;      /* output byte buffer */
+    size_t buf_len;
+    size_t buf_cap;
+    int bit_buf;       /* accumulates 8 bits before flushing a byte */
+    int bit_count;     /* bits in bit_buf (0..7) */
+} ArithEncoder;
+
+/* Arithmetic decoder */
+typedef struct {
+    const uint8_t *data;
+    size_t data_len;
+    size_t bit_pos;
+    uint32_t low;
+    uint32_t high;
+    uint32_t value;
+} ArithDecoder;
+
+/* ── Encoder ── */
+
+static inline void ae_init(ArithEncoder *e) {
+    e->low = 0;
+    e->high = 0xFFFFFFFF;
+    e->pending = 0;
+    e->buf_cap = 4096;
+    e->buf_len = 0;
+    e->buf = (uint8_t *)malloc(e->buf_cap);
+    e->bit_buf = 0;
+    e->bit_count = 0;
+}
+
+static inline void ae_flush_byte(ArithEncoder *e) {
+    if (e->buf_len >= e->buf_cap) {
+        e->buf_cap *= 2;
+        e->buf = (uint8_t *)realloc(e->buf, e->buf_cap);
+    }
+    e->buf[e->buf_len++] = (uint8_t)e->bit_buf;
+    e->bit_buf = 0;
+    e->bit_count = 0;
+}
+
+static inline void ae_output_bit(ArithEncoder *e, int bit) {
+    e->bit_buf = (e->bit_buf << 1) | bit;
+    e->bit_count++;
+    if (e->bit_count == 8) ae_flush_byte(e);
+
+    int inv = 1 - bit;
+    while (e->pending > 0) {
+        e->bit_buf = (e->bit_buf << 1) | inv;
+        e->bit_count++;
+        if (e->bit_count == 8) ae_flush_byte(e);
+        e->pending--;
+    }
+}
+
+static inline void ae_encode(ArithEncoder *e, const int64_t *cumfreqs,
+                              int symbol, int64_t total) {
+    uint64_t rng = (uint64_t)e->high - e->low + 1;
+    e->high = e->low + (uint32_t)((rng * cumfreqs[symbol + 1]) / total) - 1;
+    e->low  = e->low + (uint32_t)((rng * cumfreqs[symbol]) / total);
+
+    for (;;) {
+        if (e->high < 0x80000000u) {
+            ae_output_bit(e, 0);
+        } else if (e->low >= 0x80000000u) {
+            ae_output_bit(e, 1);
+            e->low  -= 0x80000000u;
+            e->high -= 0x80000000u;
+        } else if (e->low >= 0x40000000u && e->high < 0xC0000000u) {
+            e->pending++;
+            e->low  -= 0x40000000u;
+            e->high -= 0x40000000u;
+        } else {
+            break;
+        }
+        e->low  = (e->low << 1) & 0xFFFFFFFF;
+        e->high = ((e->high << 1) | 1) & 0xFFFFFFFF;
+    }
+}
+
+static inline void ae_finish(ArithEncoder *e) {
+    e->pending++;
+    if (e->low < 0x40000000u)
+        ae_output_bit(e, 0);
+    else
+        ae_output_bit(e, 1);
+
+    /* pad remaining bits in the last byte */
+    if (e->bit_count > 0) {
+        e->bit_buf <<= (8 - e->bit_count);
+        ae_flush_byte(e);
+    }
+}
+
+static inline void ae_free(ArithEncoder *e) {
+    free(e->buf);
+    e->buf = NULL;
+}
+
+/* ── Decoder ── */
+
+static inline int ad_read_bit(ArithDecoder *d) {
+    size_t byte_idx = d->bit_pos / 8;
+    if (byte_idx >= d->data_len) {
+        d->bit_pos++;
+        return 0;
+    }
+    int bit = (d->data[byte_idx] >> (7 - (d->bit_pos % 8))) & 1;
+    d->bit_pos++;
+    return bit;
+}
+
+static inline void ad_init(ArithDecoder *d, const uint8_t *data, size_t len) {
+    d->data = data;
+    d->data_len = len;
+    d->bit_pos = 0;
+    d->low = 0;
+    d->high = 0xFFFFFFFF;
+    d->value = 0;
+    for (int i = 0; i < 32; i++)
+        d->value = (d->value << 1) | ad_read_bit(d);
+}
+
+static inline int ad_decode(ArithDecoder *d, const int64_t *cumfreqs,
+                             int64_t total) {
+    uint64_t rng = (uint64_t)d->high - d->low + 1;
+    int64_t scaled = (int64_t)(((uint64_t)(d->value - d->low + 1) * total - 1) / rng);
+
+    /* linear search (matches Python behavior) */
+    int sym = 0;
+    for (sym = 0; sym < 256; sym++) {
+        if (cumfreqs[sym + 1] > scaled)
+            break;
+    }
+
+    d->high = d->low + (uint32_t)((rng * cumfreqs[sym + 1]) / total) - 1;
+    d->low  = d->low + (uint32_t)((rng * cumfreqs[sym]) / total);
+
+    for (;;) {
+        if (d->high < 0x80000000u) {
+            /* nothing */
+        } else if (d->low >= 0x80000000u) {
+            d->low   -= 0x80000000u;
+            d->high  -= 0x80000000u;
+            d->value -= 0x80000000u;
+        } else if (d->low >= 0x40000000u && d->high < 0xC0000000u) {
+            d->low   -= 0x40000000u;
+            d->high  -= 0x40000000u;
+            d->value -= 0x40000000u;
+        } else {
+            break;
+        }
+        d->low   = (d->low << 1) & 0xFFFFFFFF;
+        d->high  = ((d->high << 1) | 1) & 0xFFFFFFFF;
+        d->value = ((d->value << 1) | ad_read_bit(d)) & 0xFFFFFFFF;
+    }
+    return sym;
+}
+
+#endif /* ARITH_H */

delta_vs_noise.c

@@ -0,0 +1,226 @@
+/*
+ * delta_vs_noise.c — Experiment: |δ| vs noise level γ
+ *
+ * Compresses a file using the full pipeline, then dumps the
+ * calibration table statistics showing mean |δ| per confidence bin.
+ *
+ * Usage: ./delta_vs_noise <input_file>
+ *
+ * Output: TSV table of (conf_bin, γ_approx, mean_|δ|, weighted_mean_|δ|, total_obs)
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+
+#include "fastmath.h"
+#include "arith.h"
+#include "ppm.h"
+#include "tweedie.h"
+#include "match.h"
+#include "word.h"
+#include "highctx.h"
+
+#define SCALE (1 << 14)
+
+static void probs_to_cumfreqs(const double *probs, int64_t *cumfreqs,
+                               int64_t *out_total) {
+    cumfreqs[0] = 0;
+    for (int i = 0; i < 256; i++) {
+        int64_t f = (int64_t)(probs[i] * SCALE + 0.5);
+        if (f < 1) f = 1;
+        cumfreqs[i + 1] = cumfreqs[i] + f;
+    }
+    *out_total = cumfreqs[256];
+}
+
+static void clamp_normalize(double *probs) {
+    double sum = 0.0;
+    for (int i = 0; i < 256; i++) {
+        if (probs[i] < 1e-10) probs[i] = 1e-10;
+        sum += probs[i];
+    }
+    double inv = 1.0 / sum;
+    for (int i = 0; i < 256; i++)
+        probs[i] *= inv;
+}
+
+/* Representative C value for each confidence bin.
+ * twd_conf_bin uses: bin = (int)(ln(C) / 1.3863)
+ * with C < 4 → bin 0.
+ * Bin boundaries: 0:[0,4), 1:[4,e^1.39)≈[4,16), 2:[16,59), ... */
+static double conf_bin_representative_C(int bin) {
+    if (bin == 0) return 2.0;    /* midpoint of [0, 4) */
+    /* bin = floor(ln(C) / 1.3863), so midpoint is exp((bin + 0.5) * 1.3863) */
+    return exp((bin + 0.5) * 1.3862943611198906);
+}
+
+int main(int argc, char **argv) {
+    if (argc < 2) {
+        fprintf(stderr, "Usage: %s <input_file>\n", argv[0]);
+        return 1;
+    }
+
+    FILE *fin = fopen(argv[1], "rb");
+    if (!fin) { perror(argv[1]); return 1; }
+    fseek(fin, 0, SEEK_END);
+    long file_size = ftell(fin);
+    fseek(fin, 0, SEEK_SET);
+    uint8_t *data = (uint8_t *)malloc(file_size);
+    if (fread(data, 1, file_size, fin) != (size_t)file_size) {
+        fprintf(stderr, "Read error\n"); fclose(fin); return 1;
+    }
+    fclose(fin);
+
+    fprintf(stderr, "Processing %s (%ld bytes)...\n", argv[1], file_size);
+
+    /* Run the full pipeline to populate calibration tables */
+    PPMModel ppm;       ppm_init(&ppm);
+    MatchModel match;   match_init(&match);
+    WordModel word;     word_init(&word);
+    HighCtxModel hctx;  highctx_init(&hctx);
+    ArithEncoder enc;   ae_init(&enc);
+    TweedieDenoiser twd; tweedie_init(&twd);
+
+    double probs[256], word_probs[256], hctx_probs[256];
+    int64_t cumfreqs[257];
+    int64_t total;
+
+    struct timespec t0, t1;
+    clock_gettime(CLOCK_MONOTONIC, &t0);
+
+    for (long i = 0; i < file_size; i++) {
+        uint8_t byte = data[i];
+
+        double confidence;
+        int order;
+        ppm_predict(&ppm, probs, &confidence, &order);
+
+        tweedie_denoise(&twd, probs, order, confidence);
+        clamp_normalize(probs);
+
+        int match_byte;
+        double match_conf;
+        match_predict(&match, &match_byte, &match_conf);
+        blend_match(probs, match_byte, match_conf);
+
+        double w_conf;
+        if (word_predict_cached(&word, word_probs, &w_conf))
+            blend_word_model(probs, word_probs, w_conf);
+
+        double hctx_conf;
+        if (highctx_predict(&hctx, hctx_probs, &hctx_conf))
+            blend_highctx(probs, hctx_probs, hctx_conf);
+
+        probs_to_cumfreqs(probs, cumfreqs, &total);
+        ae_encode(&enc, cumfreqs, byte, total);
+
+        tweedie_update(&twd, byte);
+        match_update(&match, byte);
+        word_update(&word, byte);
+        highctx_update(&hctx, byte);
+        ppm_update(&ppm, byte);
+
+        if ((i + 1) % 50000 == 0) {
+            clock_gettime(CLOCK_MONOTONIC, &t1);
+            double elapsed = (t1.tv_sec - t0.tv_sec) + (t1.tv_nsec - t0.tv_nsec) * 1e-9;
+            fprintf(stderr, "\r  %5.1f%%  (%.0f B/s)",
+                    (i + 1) * 100.0 / file_size, (i + 1) / elapsed);
+        }
+    }
+
+    clock_gettime(CLOCK_MONOTONIC, &t1);
+    double elapsed = (t1.tv_sec - t0.tv_sec) + (t1.tv_nsec - t0.tv_nsec) * 1e-9;
+    fprintf(stderr, "\r  Done in %.1fs                    \n", elapsed);
+
+    /* ── Analyze calibration tables: mean |δ| per confidence bin ── */
+
+    /* Aggregate across all steps, bit contexts, order groups, shapes, prob bins */
+    double sum_abs_delta[TWD_N_CONF];
+    double sum_weight[TWD_N_CONF];
+    double sum_weighted_abs_delta[TWD_N_CONF];
+    int    count[TWD_N_CONF];
+    memset(sum_abs_delta, 0, sizeof(sum_abs_delta));
+    memset(sum_weight, 0, sizeof(sum_weight));
+    memset(sum_weighted_abs_delta, 0, sizeof(sum_weighted_abs_delta));
+    memset(count, 0, sizeof(count));
+
+    for (int t = 0; t < TWD_STEPS; t++)
+        for (int b = 0; b < TWD_N_BCTX; b++)
+            for (int o = 0; o < TWD_N_ORD; o++)
+                for (int s = 0; s < TWD_N_SHAPE; s++)
+                    for (int c = 0; c < TWD_N_CONF; c++)
+                        for (int p = 0; p < TWD_N_PROB; p++) {
+                            TwdCalibEntry *e = &twd.table[t][b][o][s][c][p];
+                            double real_obs = e->total - TWD_PRIOR_WEIGHT;
+                            if (real_obs < 1.0) continue;  /* skip bins with only prior */
+
+                            double avg_pred = e->sum_pred / e->total;
+                            double emp_rate = e->hits / e->total;
+                            double delta = emp_rate - avg_pred;
+
+                            sum_abs_delta[c] += fabs(delta);
+                            sum_weighted_abs_delta[c] += fabs(delta) * real_obs;
+                            sum_weight[c] += real_obs;
+                            count[c]++;
+                        }
+
+    /* ── Output ── */
+    printf("# Delta vs Noise Level — %s (%ld bytes)\n", argv[1], file_size);
+    printf("# conf_bin\tC_repr\tgamma\tmean_abs_delta\tweighted_abs_delta\tactive_bins\ttotal_obs\n");
+
+    for (int c = 0; c < TWD_N_CONF; c++) {
+        double C_repr = conf_bin_representative_C(c);
+        double gamma = 128.0 / (C_repr + 128.0);
+        double mean_d = (count[c] > 0) ? sum_abs_delta[c] / count[c] : 0.0;
+        double wmean_d = (sum_weight[c] > 0) ? sum_weighted_abs_delta[c] / sum_weight[c] : 0.0;
+
+        printf("%d\t%.1f\t%.4f\t%.6f\t%.6f\t%d\t%.0f\n",
+               c, C_repr, gamma, mean_d, wmean_d, count[c], sum_weight[c]);
+    }
+
+    /* ── Also output per-step breakdown ── */
+    printf("\n# Per-step breakdown:\n");
+    printf("# step\tconf_bin\tgamma\tweighted_abs_delta\ttotal_obs\n");
+
+    for (int t = 0; t < TWD_STEPS; t++) {
+        double step_sum_wd[TWD_N_CONF] = {0};
+        double step_sum_w[TWD_N_CONF] = {0};
+
+        for (int b = 0; b < TWD_N_BCTX; b++)
+            for (int o = 0; o < TWD_N_ORD; o++)
+                for (int s = 0; s < TWD_N_SHAPE; s++)
+                    for (int c = 0; c < TWD_N_CONF; c++)
+                        for (int p = 0; p < TWD_N_PROB; p++) {
+                            TwdCalibEntry *e = &twd.table[t][b][o][s][c][p];
+                            double real_obs = e->total - TWD_PRIOR_WEIGHT;
+                            if (real_obs < 1.0) continue;
+
+                            double avg_pred = e->sum_pred / e->total;
+                            double emp_rate = e->hits / e->total;
+                            double delta = emp_rate - avg_pred;
+
+                            step_sum_wd[c] += fabs(delta) * real_obs;
+                            step_sum_w[c] += real_obs;
+                        }
+
+        for (int c = 0; c < TWD_N_CONF; c++) {
+            double C_repr = conf_bin_representative_C(c);
+            double gamma = 128.0 / (C_repr + 128.0);
+            double wmean_d = (step_sum_w[c] > 0) ? step_sum_wd[c] / step_sum_w[c] : 0.0;
+            printf("%d\t%d\t%.4f\t%.6f\t%.0f\n",
+                   t, c, gamma, wmean_d, step_sum_w[c]);
+        }
+    }
+
+    free(data);
+    ppm_free(&ppm);
+    match_free(&match);
+    word_free(&word);
+    highctx_free(&hctx);
+    ae_free(&enc);
+
+    return 0;
+}

fastmath.h

@@ -0,0 +1,88 @@
+#ifndef FASTMATH_H
+#define FASTMATH_H
+
+#include <stdint.h>
+#include <math.h>
+
+/*
+ * Fast log/exp approximations using IEEE 754 bit tricks + polynomial correction.
+ * Accurate to ~1e-4 relative error — sufficient for probability manipulation.
+ */
+
+/* Fast natural log. Relative error < 2e-4 over [1e-30, 1.0] */
+static inline double fast_log(double x) {
+    union { double d; uint64_t u; } v = { .d = x };
+    /* Extract exponent and mantissa from IEEE 754 */
+    int64_t exp_bits = (int64_t)((v.u >> 52) & 0x7FF) - 1023;
+    /* Set exponent to 0 → mantissa in [1, 2) */
+    v.u = (v.u & 0x000FFFFFFFFFFFFFULL) | 0x3FF0000000000000ULL;
+    double m = v.d;
+    /* Polynomial approx of log(m) for m in [1,2): Remez-like */
+    /* log(m) ≈ (m-1) - (m-1)^2/2 + (m-1)^3/3 ... but use minimax */
+    double t = m - 1.0;
+    double log_m = t * (1.0 + t * (-0.5 + t * (0.333333333 + t * (-0.25 + t * 0.2))));
+    return log_m + exp_bits * 0.6931471805599453; /* exp_bits * ln(2) */
+}
+
+/* Fast exp. Relative error < 3e-4 over [-90, 0] (typical range for log-probs) */
+static inline double fast_exp(double x) {
+    if (x < -700.0) return 0.0;
+    if (x > 709.0) return 1e308;
+    /* exp(x) = 2^(x/ln2) = 2^(k+f) where k=floor, f=frac */
+    double t = x * 1.4426950408889634; /* x / ln(2) */
+    int64_t k = (int64_t)t;
+    if (t < k) k--; /* floor for negative */
+    double f = t - k;
+    /* 2^f for f in [0,1): minimax polynomial */
+    double p = 1.0 + f * (0.6931471805599453 + f * (0.24022650695910071
+              + f * (0.05550410866482158 + f * (0.009618129107628477
+              + f * 0.0013333558146428443))));
+    /* Multiply by 2^k via bit manipulation */
+    union { double d; uint64_t u; } v;
+    v.u = (uint64_t)(k + 1023) << 52;
+    return p * v.d;
+}
+
+/* Fast log(a / (1-a)) — logit function via single IEEE bit trick.
+ * Uses the identity: logit(p) = log(p) - log(1-p)
+ * We can compute log(p/(1-p)) in one pass by exploiting IEEE 754. */
+static inline double fast_logit(double p) {
+    if (p < 1e-8) p = 1e-8;
+    if (p > 1.0 - 1e-8) p = 1.0 - 1e-8;
+    /* For p near 0.5, use rational approx; otherwise use fast_log */
+    double r = p / (1.0 - p);
+    return fast_log(r);
+}
+
+/*
+ * Precomputed logit lookup table for probabilities.
+ * Maps probability [0..65536]/65536 → logit value.
+ * Avoids per-symbol log computation entirely.
+ */
+#define LOGIT_TABLE_SIZE 65537
+typedef struct {
+    double table[LOGIT_TABLE_SIZE];
+    int initialized;
+} LogitTable;
+
+static inline void logit_table_init(LogitTable *lt) {
+    for (int i = 0; i < LOGIT_TABLE_SIZE; i++) {
+        double p = (double)i / 65536.0;
+        if (p < 1e-8) p = 1e-8;
+        if (p > 1.0 - 1e-8) p = 1.0 - 1e-8;
+        lt->table[i] = log(p / (1.0 - p));
+    }
+    lt->initialized = 1;
+}
+
+static inline double logit_table_lookup(const LogitTable *lt, double p) {
+    int idx = (int)(p * 65536.0 + 0.5);
+    if (idx < 0) idx = 0;
+    if (idx >= LOGIT_TABLE_SIZE) idx = LOGIT_TABLE_SIZE - 1;
+    return lt->table[idx];
+}
+
+/* Fast sqrt (just use hardware — it's already fast) */
+/* static inline double fast_sqrt(double x) { return sqrt(x); } */
+
+#endif /* FASTMATH_H */

highctx.h

@@ -0,0 +1,208 @@
+#ifndef HIGHCTX_H
+#define HIGHCTX_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+/*
+ * High-Order Context Model (orders 5-8)
+ *
+ * Extends effective context beyond PPM's order-4 limit without modifying PPM.
+ * Uses hash tables mapping context_hash → count[256] for orders 5, 6, 7, 8.
+ * Unlike the match model (which finds one position, predicts one byte),
+ * this aggregates ALL matching positions into a full probability distribution.
+ *
+ * Blended after SSE in the pipeline, preserving diffusion's contribution.
+ */
+
+#define HCTX_NSYM    256
+#define HCTX_N_ORDERS 4   /* orders 5, 6, 7, 8 */
+#define HCTX_MIN_ORDER 5
+#define HCTX_MAX_ENTRIES (1 << 20)  /* 1M entries per table, ~500MB total max */
+
+/* Hash table entry: context hash → byte counts */
+typedef struct {
+    uint64_t key;
+    uint16_t counts[HCTX_NSYM];
+    uint32_t total;
+} HCtxEntry;
+
+typedef struct {
+    HCtxEntry *entries;
+    uint32_t capacity;
+    uint32_t mask;
+    uint32_t used;
+} HCtxTable;
+
+static inline void hctx_table_init(HCtxTable *t, uint32_t cap) {
+    t->capacity = cap;
+    t->mask = cap - 1;
+    t->used = 0;
+    t->entries = (HCtxEntry *)calloc(cap, sizeof(HCtxEntry));
+}
+
+static inline void hctx_table_free(HCtxTable *t) {
+    free(t->entries);
+    t->entries = NULL;
+}
+
+static inline void hctx_table_grow(HCtxTable *t) {
+    uint32_t old_cap = t->capacity;
+    HCtxEntry *old = t->entries;
+    uint32_t new_cap = old_cap * 2;
+    t->entries = (HCtxEntry *)calloc(new_cap, sizeof(HCtxEntry));
+    t->capacity = new_cap;
+    t->mask = new_cap - 1;
+    t->used = 0;
+    for (uint32_t i = 0; i < old_cap; i++) {
+        if (old[i].key != 0) {
+            uint32_t idx = (uint32_t)(old[i].key & t->mask);
+            while (t->entries[idx].key != 0)
+                idx = (idx + 1) & t->mask;
+            t->entries[idx] = old[i];
+            t->used++;
+        }
+    }
+    free(old);
+}
+
+/* Find or create entry. Returns pointer to entry (NULL if full and create). */
+static inline HCtxEntry *hctx_table_get(HCtxTable *t, uint64_t key, int create) {
+    if (create && t->used * 5 > t->capacity * 3) {
+        if (t->capacity < HCTX_MAX_ENTRIES)
+            hctx_table_grow(t);
+        else
+            create = 0;  /* at max capacity, only look up existing */
+    }
+    uint32_t idx = (uint32_t)(key & t->mask);
+    for (;;) {
+        if (t->entries[idx].key == key)
+            return &t->entries[idx];
+        if (t->entries[idx].key == 0) {
+            if (!create) return NULL;
+            t->entries[idx].key = key;
+            memset(t->entries[idx].counts, 0, sizeof(t->entries[idx].counts));
+            t->entries[idx].total = 0;
+            t->used++;
+            return &t->entries[idx];
+        }
+        idx = (idx + 1) & t->mask;
+    }
+}
+
+/* FNV-1a hash for context bytes */
+static inline uint64_t hctx_hash(const uint8_t *data, int len) {
+    uint64_t h = 14695981039346656037ULL;
+    for (int i = 0; i < len; i++) {
+        h ^= data[i];
+        h *= 1099511628211ULL;
+    }
+    if (h == 0) h = 1;
+    return h;
+}
+
+typedef struct {
+    HCtxTable tables[HCTX_N_ORDERS];  /* orders 5, 6, 7, 8 */
+    uint8_t *history;
+    int hist_len;
+    int hist_cap;
+} HighCtxModel;
+
+static inline void highctx_init(HighCtxModel *m) {
+    for (int i = 0; i < HCTX_N_ORDERS; i++)
+        hctx_table_init(&m->tables[i], 8192);
+    m->hist_cap = 4096;
+    m->hist_len = 0;
+    m->history = (uint8_t *)malloc(m->hist_cap);
+}
+
+static inline void highctx_free(HighCtxModel *m) {
+    for (int i = 0; i < HCTX_N_ORDERS; i++)
+        hctx_table_free(&m->tables[i]);
+    free(m->history);
+    m->history = NULL;
+}
+
+/*
+ * Predict: try highest order first (8, 7, 6, 5).
+ * Use the highest order that has a context with total >= min_count.
+ * Returns 1 if prediction available, fills probs[256] and *out_conf.
+ */
+static inline int highctx_predict(HighCtxModel *m, double *probs, double *out_conf) {
+    int n = m->hist_len;
+
+    for (int oidx = HCTX_N_ORDERS - 1; oidx >= 0; oidx--) {
+        int order = HCTX_MIN_ORDER + oidx;  /* 8, 7, 6, 5 */
+        if (n < order) continue;
+
+        uint64_t key = hctx_hash(m->history + n - order, order);
+        HCtxEntry *e = hctx_table_get(&m->tables[oidx], key, 0);
+        if (!e || e->total < 4) continue;
+
+        /* Build distribution: sparse smoothing to avoid zero probs */
+        double smooth = 1e-4;
+        double total_smooth = e->total + smooth * HCTX_NSYM;
+        double inv = 1.0 / total_smooth;
+        for (int s = 0; s < HCTX_NSYM; s++)
+            probs[s] = (e->counts[s] + smooth) * inv;
+
+        /* Confidence: ramps slowly, requires real data */
+        double count_conf = (e->total - 4.0) / (e->total + 8.0);  /* 0 at total=4, ~0.7 at 20 */
+        if (count_conf < 0) count_conf = 0;
+        double order_factor = 0.4 + (order - HCTX_MIN_ORDER) * 0.1;  /* 0.4 for o5, 0.7 for o8 */
+        *out_conf = count_conf * order_factor;
+
+        return 1;
+    }
+
+    *out_conf = 0.0;
+    return 0;
+}
+
+/*
+ * Update: increment counts for all available orders.
+ */
+static inline void highctx_update(HighCtxModel *m, uint8_t byte) {
+    int n = m->hist_len;
+
+    for (int oidx = 0; oidx < HCTX_N_ORDERS; oidx++) {
+        int order = HCTX_MIN_ORDER + oidx;
+        if (n >= order) {
+            uint64_t key = hctx_hash(m->history + n - order, order);
+            HCtxEntry *e = hctx_table_get(&m->tables[oidx], key, 1);
+            if (e) {
+                e->counts[byte]++;
+                e->total++;
+            }
+        }
+    }
+
+    /* Append to history */
+    if (m->hist_len >= m->hist_cap) {
+        m->hist_cap *= 2;
+        m->history = (uint8_t *)realloc(m->history, m->hist_cap);
+    }
+    m->history[m->hist_len++] = byte;
+}
+
+/*
+ * Blend high-context prediction into existing probability distribution.
+ */
+static inline void blend_highctx(double *probs, const double *hctx_probs,
+                                   double hctx_conf) {
+    if (hctx_conf < 0.01) return;
+    double weight = hctx_conf * 2.0;
+    if (weight > 0.60) weight = 0.60;
+    double sum = 0.0;
+    for (int i = 0; i < HCTX_NSYM; i++) {
+        probs[i] = probs[i] * (1.0 - weight) + hctx_probs[i] * weight;
+        if (probs[i] < 1e-10) probs[i] = 1e-10;
+        sum += probs[i];
+    }
+    double inv = 1.0 / sum;
+    for (int i = 0; i < HCTX_NSYM; i++)
+        probs[i] *= inv;
+}
+
+#endif /* HIGHCTX_H */

match.h

@@ -0,0 +1,234 @@
+#ifndef MATCH_H
+#define MATCH_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define MATCH_NSYM 256
+#define MATCH_N_CTX 5  /* context lengths: 4, 6, 8, 12, 16 */
+
+/* Hash table entry: context hash → position in history */
+typedef struct {
+    uint64_t key;
+    uint32_t pos;
+} MatchHTEntry;
+
+typedef struct {
+    MatchHTEntry *entries;
+    uint32_t capacity;
+    uint32_t mask;
+} MatchHT;
+
+static inline void mht_init(MatchHT *t, uint32_t cap) {
+    t->capacity = cap;
+    t->mask = cap - 1;
+    t->entries = (MatchHTEntry *)calloc(cap, sizeof(MatchHTEntry));
+}
+
+static inline void mht_free(MatchHT *t) {
+    free(t->entries);
+}
+
+static inline void mht_grow(MatchHT *t) {
+    uint32_t old_cap = t->capacity;
+    MatchHTEntry *old = t->entries;
+    uint32_t new_cap = old_cap * 2;
+    t->entries = (MatchHTEntry *)calloc(new_cap, sizeof(MatchHTEntry));
+    t->capacity = new_cap;
+    t->mask = new_cap - 1;
+    for (uint32_t i = 0; i < old_cap; i++) {
+        if (old[i].key != 0) {
+            uint32_t idx = (uint32_t)(old[i].key & t->mask);
+            while (t->entries[idx].key != 0)
+                idx = (idx + 1) & t->mask;
+            t->entries[idx] = old[i];
+        }
+    }
+    free(old);
+}
+
+static inline void mht_set(MatchHT *t, uint64_t key, uint32_t pos,
+                             uint32_t *used) {
+    if (*used * 5 > t->capacity * 3) mht_grow(t);
+    uint32_t idx = (uint32_t)(key & t->mask);
+    for (;;) {
+        if (t->entries[idx].key == key || t->entries[idx].key == 0) {
+            if (t->entries[idx].key == 0) (*used)++;
+            t->entries[idx].key = key;
+            t->entries[idx].pos = pos;
+            return;
+        }
+        idx = (idx + 1) & t->mask;
+    }
+}
+
+static inline int mht_get(MatchHT *t, uint64_t key, uint32_t *out_pos) {
+    uint32_t idx = (uint32_t)(key & t->mask);
+    for (;;) {
+        if (t->entries[idx].key == key) {
+            *out_pos = t->entries[idx].pos;
+            return 1;
+        }
+        if (t->entries[idx].key == 0) return 0;
+        idx = (idx + 1) & t->mask;
+    }
+}
+
+typedef struct {
+    int ctx_lens[MATCH_N_CTX];
+    MatchHT tables[MATCH_N_CTX];
+    uint32_t table_used[MATCH_N_CTX];
+    uint8_t *history;
+    int hist_len;
+    int hist_cap;
+
+    /* active match state */
+    int match_read_pos;
+    int match_active;
+    int match_streak;
+
+    /* adaptive accuracy */
+    double hits;
+    double total;
+} MatchModel;
+
+static inline uint64_t match_ctx_hash(const uint8_t *data, int len) {
+    uint64_t h = 14695981039346656037ULL;
+    for (int i = 0; i < len; i++) {
+        h ^= data[i];
+        h *= 1099511628211ULL;
+    }
+    if (h == 0) h = 1;
+    return h;
+}
+
+static inline void match_init(MatchModel *m) {
+    m->ctx_lens[0] = 4;
+    m->ctx_lens[1] = 6;
+    m->ctx_lens[2] = 8;
+    m->ctx_lens[3] = 12;
+    m->ctx_lens[4] = 16;
+    for (int i = 0; i < MATCH_N_CTX; i++) {
+        mht_init(&m->tables[i], 4096);
+        m->table_used[i] = 0;
+    }
+    m->hist_cap = 4096;
+    m->hist_len = 0;
+    m->history = (uint8_t *)malloc(m->hist_cap);
+    m->match_read_pos = -1;
+    m->match_active = 0;
+    m->match_streak = 0;
+    m->hits = 1.0;
+    m->total = 2.0;
+}
+
+static inline void match_free(MatchModel *m) {
+    for (int i = 0; i < MATCH_N_CTX; i++)
+        mht_free(&m->tables[i]);
+    free(m->history);
+    m->history = NULL;
+}
+
+/*
+ * predict: returns predicted byte via *out_byte, confidence via *out_conf.
+ * Returns 1 if prediction available, 0 otherwise.
+ */
+static inline int match_predict(MatchModel *m, int *out_byte, double *out_conf) {
+    /* 1. Continue active match */
+    if (m->match_active && m->match_read_pos >= 0
+        && m->match_read_pos < m->hist_len) {
+        *out_byte = m->history[m->match_read_pos];
+        double base = m->hits / m->total;
+        double conf = base * (0.65 + m->match_streak * 0.04);
+        if (conf > 0.96) conf = 0.96;
+        *out_conf = conf;
+        return 1;
+    }
+
+    m->match_active = 0;
+
+    /* 2. Try new match (longest context first) */
+    for (int idx = MATCH_N_CTX - 1; idx >= 0; idx--) {
+        int ctx_len = m->ctx_lens[idx];
+        int n = m->hist_len;
+        if (n < ctx_len) continue;
+
+        uint64_t key = match_ctx_hash(m->history + n - ctx_len, ctx_len);
+        uint32_t pos;
+        if (mht_get(&m->tables[idx], key, &pos) && pos < (uint32_t)n) {
+            *out_byte = m->history[pos];
+            m->match_active = 1;
+            m->match_read_pos = (int)pos;
+            m->match_streak = 0;
+            double base = m->hits / m->total;
+            double conf = base * (ctx_len / 6.0);
+            if (conf > base * 0.9) conf = base * 0.9;
+            *out_conf = conf;
+            return 1;
+        }
+    }
+
+    *out_byte = -1;
+    *out_conf = 0.0;
+    return 0;
+}
+
+static inline void match_update(MatchModel *m, uint8_t actual_byte) {
+    /* track accuracy of active match */
+    if (m->match_active && m->match_read_pos >= 0
+        && m->match_read_pos < m->hist_len) {
+        int predicted = m->history[m->match_read_pos];
+        m->total += 1.0;
+        if (predicted == actual_byte) {
+            m->hits += 1.0;
+            m->match_streak++;
+            m->match_read_pos++;
+        } else {
+            m->match_active = 0;
+            m->match_streak = 0;
+        }
+        if (m->total > 500.0) {
+            m->hits *= 0.99;
+            m->total *= 0.99;
+        }
+    }
+
+    /* store context → position */
+    int n = m->hist_len;
+    for (int tidx = 0; tidx < MATCH_N_CTX; tidx++) {
+        int ctx_len = m->ctx_lens[tidx];
+        if (n >= ctx_len) {
+            uint64_t key = match_ctx_hash(m->history + n - ctx_len, ctx_len);
+            mht_set(&m->tables[tidx], key, (uint32_t)n,
+                     &m->table_used[tidx]);
+        }
+    }
+
+    /* append to history */
+    if (m->hist_len >= m->hist_cap) {
+        m->hist_cap *= 2;
+        m->history = (uint8_t *)realloc(m->history, m->hist_cap);
+    }
+    m->history[m->hist_len++] = actual_byte;
+}
+
+static inline void blend_match(double *probs, int match_byte,
+                                double match_confidence) {
+    if (match_byte < 0) return;
+    double weight = match_confidence * 0.85;
+    if (weight > 0.95) weight = 0.95;
+    for (int i = 0; i < MATCH_NSYM; i++)
+        probs[i] *= (1.0 - weight);
+    probs[match_byte] += weight;
+    double sum = 0.0;
+    for (int i = 0; i < MATCH_NSYM; i++) {
+        if (probs[i] < 1e-10) probs[i] = 1e-10;
+        sum += probs[i];
+    }
+    double inv = 1.0 / sum;
+    for (int i = 0; i < MATCH_NSYM; i++)
+        probs[i] *= inv;
+}
+
+#endif /* MATCH_H */

mdc.c

@@ -0,0 +1,295 @@
+/*
+ * Midicoth Compressor — C implementation
+ * Pipeline: PPM + Tweedie Denoising + Match + Word + HighCtx
+ *
+ * Usage:
+ *   ./mdc compress   <input> <output>
+ *   ./mdc decompress <input> <output>
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+
+#include "fastmath.h"
+#include "arith.h"
+#include "ppm.h"
+#include "tweedie.h"
+#include "match.h"
+#include "word.h"
+#include "highctx.h"
+
+#define MAGIC "MDC7"
+#define SCALE (1 << 14)
+
+/* ── Helpers ── */
+
+static void probs_to_cumfreqs(const double *probs, int64_t *cumfreqs,
+                               int64_t *out_total) {
+    cumfreqs[0] = 0;
+    for (int i = 0; i < 256; i++) {
+        int64_t f = (int64_t)(probs[i] * SCALE + 0.5);
+        if (f < 1) f = 1;
+        cumfreqs[i + 1] = cumfreqs[i] + f;
+    }
+    *out_total = cumfreqs[256];
+}
+
+static void clamp_normalize(double *probs) {
+    double sum = 0.0;
+    for (int i = 0; i < 256; i++) {
+        if (probs[i] < 1e-10) probs[i] = 1e-10;
+        sum += probs[i];
+    }
+    double inv = 1.0 / sum;
+    for (int i = 0; i < 256; i++)
+        probs[i] *= inv;
+}
+
+/* ── Compress ── */
+
+static int do_compress(const char *input_path, const char *output_path) {
+    FILE *fin = fopen(input_path, "rb");
+    if (!fin) { perror(input_path); return 1; }
+    fseek(fin, 0, SEEK_END);
+    long file_size = ftell(fin);
+    fseek(fin, 0, SEEK_SET);
+    uint8_t *data = (uint8_t *)malloc(file_size);
+    if (fread(data, 1, file_size, fin) != (size_t)file_size) {
+        fprintf(stderr, "Read error\n"); fclose(fin); return 1;
+    }
+    fclose(fin);
+
+    uint64_t original_size = (uint64_t)file_size;
+    printf("  Input:  %s (%lu bytes)\n", input_path, (unsigned long)original_size);
+
+    if (original_size == 0) {
+        FILE *fout = fopen(output_path, "wb");
+        fwrite(MAGIC, 1, 4, fout);
+        uint64_t zero = 0;
+        fwrite(&zero, 8, 1, fout);
+        fclose(fout);
+        printf("  Empty file -> 12 bytes\n");
+        free(data);
+        return 0;
+    }
+
+    PPMModel ppm;     ppm_init(&ppm);
+    MatchModel match; match_init(&match);
+    WordModel word;   word_init(&word);
+    HighCtxModel hctx; highctx_init(&hctx);
+    ArithEncoder enc; ae_init(&enc);
+    TweedieDenoiser twd; tweedie_init(&twd);
+
+    double probs[256], word_probs[256], hctx_probs[256];
+    int64_t cumfreqs[257];
+    int64_t total;
+
+    struct timespec t0, t1;
+    clock_gettime(CLOCK_MONOTONIC, &t0);
+
+    for (uint64_t i = 0; i < original_size; i++) {
+        uint8_t byte = data[i];
+
+        double confidence;
+        int order;
+        ppm_predict(&ppm, probs, &confidence, &order);
+
+        tweedie_denoise(&twd, probs, order, confidence);
+        clamp_normalize(probs);
+
+        int match_byte;
+        double match_conf;
+        match_predict(&match, &match_byte, &match_conf);
+        blend_match(probs, match_byte, match_conf);
+
+        double w_conf;
+        if (word_predict_cached(&word, word_probs, &w_conf))
+            blend_word_model(probs, word_probs, w_conf);
+
+        double hctx_conf;
+        if (highctx_predict(&hctx, hctx_probs, &hctx_conf))
+            blend_highctx(probs, hctx_probs, hctx_conf);
+
+        probs_to_cumfreqs(probs, cumfreqs, &total);
+        ae_encode(&enc, cumfreqs, byte, total);
+
+        tweedie_update(&twd, byte);
+        match_update(&match, byte);
+        word_update(&word, byte);
+        highctx_update(&hctx, byte);
+        ppm_update(&ppm, byte);
+
+        if ((i + 1) % 50000 == 0) {
+            clock_gettime(CLOCK_MONOTONIC, &t1);
+            double elapsed = (t1.tv_sec - t0.tv_sec) + (t1.tv_nsec - t0.tv_nsec) * 1e-9;
+            double pct = (i + 1) * 100.0 / original_size;
+            double speed = (i + 1) / elapsed;
+            fprintf(stderr, "\r    %5.1f%%  (%lu/%lu)  %.0f B/s",
+                    pct, (unsigned long)(i + 1), (unsigned long)original_size, speed);
+        }
+    }
+
+    ae_finish(&enc);
+
+    clock_gettime(CLOCK_MONOTONIC, &t1);
+    double elapsed = (t1.tv_sec - t0.tv_sec) + (t1.tv_nsec - t0.tv_nsec) * 1e-9;
+    fprintf(stderr, "\r                                                \r");
+
+    FILE *fout = fopen(output_path, "wb");
+    if (!fout) { perror(output_path); return 1; }
+    fwrite(MAGIC, 1, 4, fout);
+    fwrite(&original_size, 8, 1, fout);
+    fwrite(enc.buf, 1, enc.buf_len, fout);
+    fclose(fout);
+
+    uint64_t total_size = 4 + 8 + enc.buf_len;
+    double ratio = (double)total_size / original_size;
+    printf("  Output: %s (%lu bytes)\n", output_path, (unsigned long)total_size);
+    printf("  Ratio:  %.4f (%.2f%%)\n", ratio, ratio * 100.0);
+    printf("  Time:   %.1fs (%.0f B/s)\n", elapsed, original_size / elapsed);
+
+    ae_free(&enc);
+    ppm_free(&ppm);
+    match_free(&match);
+    word_free(&word);
+    highctx_free(&hctx);
+    free(data);
+    return 0;
+}
+
+/* ── Decompress ── */
+
+static int do_decompress(const char *input_path, const char *output_path) {
+    FILE *fin = fopen(input_path, "rb");
+    if (!fin) { perror(input_path); return 1; }
+
+    char magic[4];
+    if (fread(magic, 1, 4, fin) != 4 || memcmp(magic, MAGIC, 4) != 0) {
+        fprintf(stderr, "Error: not a MDC7 file\n");
+        fclose(fin);
+        return 1;
+    }
+
+    uint64_t original_size;
+    if (fread(&original_size, 8, 1, fin) != 1) {
+        fprintf(stderr, "Read error\n"); fclose(fin); return 1;
+    }
+
+    fseek(fin, 0, SEEK_END);
+    long fsize = ftell(fin);
+    fseek(fin, 12, SEEK_SET);
+    size_t comp_len = (size_t)(fsize - 12);
+    uint8_t *compressed = (uint8_t *)malloc(comp_len);
+    if (fread(compressed, 1, comp_len, fin) != comp_len) {
+        fprintf(stderr, "Read error\n"); fclose(fin); return 1;
+    }
+    fclose(fin);
+
+    printf("  Input:  %s (%ld bytes)\n", input_path, fsize);
+    printf("  Original size: %lu bytes\n", (unsigned long)original_size);
+
+    if (original_size == 0) {
+        FILE *fout = fopen(output_path, "wb");
+        fclose(fout);
+        printf("  Empty file\n");
+        free(compressed);
+        return 0;
+    }
+
+    PPMModel ppm;     ppm_init(&ppm);
+    MatchModel match; match_init(&match);
+    WordModel word;   word_init(&word);
+    HighCtxModel hctx; highctx_init(&hctx);
+    ArithDecoder dec; ad_init(&dec, compressed, comp_len);
+    TweedieDenoiser twd; tweedie_init(&twd);
+
+    uint8_t *result = (uint8_t *)malloc(original_size);
+
+    double probs[256], word_probs[256], hctx_probs[256];
+    int64_t cumfreqs[257];
+    int64_t total;
+
+    struct timespec t0, t1;
+    clock_gettime(CLOCK_MONOTONIC, &t0);
+
+    for (uint64_t i = 0; i < original_size; i++) {
+        double confidence;
+        int order;
+        ppm_predict(&ppm, probs, &confidence, &order);
+
+        tweedie_denoise(&twd, probs, order, confidence);
+        clamp_normalize(probs);
+
+        int match_byte;
+        double match_conf;
+        match_predict(&match, &match_byte, &match_conf);
+        blend_match(probs, match_byte, match_conf);
+
+        double w_conf;
+        if (word_predict_cached(&word, word_probs, &w_conf))
+            blend_word_model(probs, word_probs, w_conf);
+
+        double hctx_conf;
+        if (highctx_predict(&hctx, hctx_probs, &hctx_conf))
+            blend_highctx(probs, hctx_probs, hctx_conf);
+
+        probs_to_cumfreqs(probs, cumfreqs, &total);
+        int sym = ad_decode(&dec, cumfreqs, total);
+        result[i] = (uint8_t)sym;
+
+        tweedie_update(&twd, (uint8_t)sym);
+        match_update(&match, (uint8_t)sym);
+        word_update(&word, (uint8_t)sym);
+        highctx_update(&hctx, (uint8_t)sym);
+        ppm_update(&ppm, (uint8_t)sym);
+
+        if ((i + 1) % 50000 == 0) {
+            clock_gettime(CLOCK_MONOTONIC, &t1);
+            double elapsed = (t1.tv_sec - t0.tv_sec) + (t1.tv_nsec - t0.tv_nsec) * 1e-9;
+            double pct = (i + 1) * 100.0 / original_size;
+            double speed = (i + 1) / elapsed;
+            fprintf(stderr, "\r    %5.1f%%  (%lu/%lu)  %.0f B/s",
+                    pct, (unsigned long)(i + 1), (unsigned long)original_size, speed);
+        }
+    }
+
+    clock_gettime(CLOCK_MONOTONIC, &t1);
+    double elapsed = (t1.tv_sec - t0.tv_sec) + (t1.tv_nsec - t0.tv_nsec) * 1e-9;
+    fprintf(stderr, "\r                                                \r");
+
+    FILE *fout = fopen(output_path, "wb");
+    fwrite(result, 1, original_size, fout);
+    fclose(fout);
+
+    printf("  Output: %s (%lu bytes)\n", output_path, (unsigned long)original_size);
+    printf("  Time:   %.1fs (%.0f B/s)\n", elapsed, original_size / elapsed);
+
+    ppm_free(&ppm);
+    match_free(&match);
+    word_free(&word);
+    highctx_free(&hctx);
+    free(compressed);
+    free(result);
+    return 0;
+}
+
+/* ── Main ── */
+
+int main(int argc, char **argv) {
+    if (argc != 4) {
+        fprintf(stderr, "Usage: %s compress|decompress <input> <output>\n", argv[0]);
+        return 1;
+    }
+
+    if (strcmp(argv[1], "compress") == 0)
+        return do_compress(argv[2], argv[3]);
+    else if (strcmp(argv[1], "decompress") == 0)
+        return do_decompress(argv[2], argv[3]);
+    else {
+        fprintf(stderr, "Unknown command: %s\n", argv[1]);
+        return 1;
+    }
+}

ppm.h

@@ -0,0 +1,198 @@
+#ifndef PPM_H
+#define PPM_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+#define PPM_MAX_ORDER 4
+#define PPM_NSYM 256
+#define PPM_PRIOR 0.5
+
+/*
+ * Hash table entry: maps a 64-bit context hash to a count array.
+ * counts[i] stores the (float) count for symbol i.
+ * total caches sum(counts).
+ * key == 0 means empty slot.
+ */
+typedef struct {
+    uint64_t key;          /* context hash (0 = empty) */
+    double counts[PPM_NSYM];
+    double total;
+} PPMEntry;
+
+typedef struct {
+    PPMEntry *entries;
+    uint32_t capacity;     /* power of 2 */
+    uint32_t used;
+} PPMTable;
+
+typedef struct {
+    PPMTable tables[PPM_MAX_ORDER + 1];  /* order 0..4 */
+    uint8_t *history;
+    int hist_len;
+    int hist_cap;
+} PPMModel;
+
+/* ── Hash helper ── */
+
+static inline uint64_t ppm_hash_context(const uint8_t *ctx, int len) {
+    /* We need a non-zero hash for all contexts including order-0 (empty).
+     * Use FNV-1a style. Order-0 empty context gets a fixed hash. */
+    if (len == 0) return 1;  /* special: order-0 empty context */
+    uint64_t h = 14695981039346656037ULL;
+    for (int i = 0; i < len; i++) {
+        h ^= ctx[i];
+        h *= 1099511628211ULL;
+    }
+    if (h == 0) h = 1;  /* reserve 0 for empty slot */
+    return h;
+}
+
+/* ── Table operations ── */
+
+static inline void ppm_table_init(PPMTable *t, uint32_t capacity) {
+    t->capacity = capacity;
+    t->used = 0;
+    t->entries = (PPMEntry *)calloc(capacity, sizeof(PPMEntry));
+}
+
+static inline void ppm_table_free(PPMTable *t) {
+    free(t->entries);
+    t->entries = NULL;
+}
+
+static inline void ppm_table_grow(PPMTable *t);
+
+static inline PPMEntry *ppm_table_find(PPMTable *t, uint64_t key) {
+    uint32_t mask = t->capacity - 1;
+    uint32_t idx = (uint32_t)(key & mask);
+    for (;;) {
+        PPMEntry *e = &t->entries[idx];
+        if (e->key == key) return e;
+        if (e->key == 0) return NULL;
+        idx = (idx + 1) & mask;
+    }
+}
+
+static inline PPMEntry *ppm_table_insert(PPMTable *t, uint64_t key) {
+    /* Grow if > 60% full */
+    if (t->used * 5 > t->capacity * 3) {
+        ppm_table_grow(t);
+    }
+    uint32_t mask = t->capacity - 1;
+    uint32_t idx = (uint32_t)(key & mask);
+    for (;;) {
+        PPMEntry *e = &t->entries[idx];
+        if (e->key == key) return e;  /* already exists */
+        if (e->key == 0) {
+            /* init new entry with prior */
+            e->key = key;
+            for (int i = 0; i < PPM_NSYM; i++)
+                e->counts[i] = PPM_PRIOR;
+            e->total = PPM_NSYM * PPM_PRIOR;
+            t->used++;
+            return e;
+        }
+        idx = (idx + 1) & mask;
+    }
+}
+
+static inline void ppm_table_grow(PPMTable *t) {
+    uint32_t old_cap = t->capacity;
+    PPMEntry *old = t->entries;
+    uint32_t new_cap = old_cap * 2;
+    t->entries = (PPMEntry *)calloc(new_cap, sizeof(PPMEntry));
+    t->capacity = new_cap;
+    t->used = 0;
+    for (uint32_t i = 0; i < old_cap; i++) {
+        if (old[i].key != 0) {
+            /* re-insert */
+            PPMEntry *ne = ppm_table_insert(t, old[i].key);
+            memcpy(ne->counts, old[i].counts, sizeof(old[i].counts));
+            ne->total = old[i].total;
+        }
+    }
+    free(old);
+}
+
+/* ── PPM Model ── */
+
+static inline void ppm_init(PPMModel *m) {
+    for (int o = 0; o <= PPM_MAX_ORDER; o++)
+        ppm_table_init(&m->tables[o], 1024);
+    m->hist_cap = 4096;
+    m->hist_len = 0;
+    m->history = (uint8_t *)malloc(m->hist_cap);
+}
+
+static inline void ppm_free(PPMModel *m) {
+    for (int o = 0; o <= PPM_MAX_ORDER; o++)
+        ppm_table_free(&m->tables[o]);
+    free(m->history);
+    m->history = NULL;
+}
+
+/*
+ * predict_with_confidence: fills probs[256] and returns confidence + order.
+ * Matches Python: fallback from max_order down to 0, first context with total > 1.
+ * If nothing found, returns uniform.
+ */
+static inline void ppm_predict(PPMModel *m, double *probs,
+                                double *out_confidence, int *out_order) {
+    for (int order = PPM_MAX_ORDER; order >= 0; order--) {
+        const uint8_t *ctx_start;
+        int ctx_len = order;
+
+        if (ctx_len > m->hist_len) continue;
+        ctx_start = m->history + m->hist_len - ctx_len;
+
+        uint64_t key = ppm_hash_context(ctx_start, ctx_len);
+        PPMEntry *e = ppm_table_find(&m->tables[order], key);
+        if (e == NULL) continue;
+        if (e->total <= 1.0) continue;
+
+        double inv_total = 1.0 / e->total;
+        for (int i = 0; i < PPM_NSYM; i++)
+            probs[i] = e->counts[i] * inv_total;
+
+        *out_confidence = e->total;
+        *out_order = order;
+        return;
+    }
+
+    /* uniform fallback */
+    double u = 1.0 / 256.0;
+    for (int i = 0; i < PPM_NSYM; i++)
+        probs[i] = u;
+    *out_confidence = 0.0;
+    *out_order = -1;
+}
+
+/*
+ * update: add symbol count to all orders (0..4) where context is available.
+ * Then append symbol to history.
+ */
+static inline void ppm_update(PPMModel *m, uint8_t symbol) {
+    for (int order = 0; order <= PPM_MAX_ORDER; order++) {
+        int ctx_len = order;
+        if (ctx_len > m->hist_len) continue;
+
+        const uint8_t *ctx_start = m->history + m->hist_len - ctx_len;
+        uint64_t key = ppm_hash_context(ctx_start, ctx_len);
+
+        PPMEntry *e = ppm_table_insert(&m->tables[order], key);
+        e->counts[symbol] += 1.0;
+        e->total += 1.0;
+    }
+
+    /* append to history */
+    if (m->hist_len >= m->hist_cap) {
+        m->hist_cap *= 2;
+        m->history = (uint8_t *)realloc(m->history, m->hist_cap);
+    }
+    m->history[m->hist_len++] = symbol;
+}
+
+#endif /* PPM_H */

tweedie.h

@@ -0,0 +1,280 @@
+#ifndef TWEEDIE_H
+#define TWEEDIE_H
+
+/*
+ * Binary Tree Tweedie Denoiser — score-based reverse diffusion.
+ *
+ * Forward process (PPM Jeffreys prior):
+ *   p̂(s) = (n·q(s) + 0.5) / (n + 128) = (1-γ)q(s) + γ·u(s)
+ *   where γ = 128/(n+128) is the noise level, u = 1/256 uniform.
+ *
+ * Tweedie's formula gives the optimal denoiser:
+ *   θ̂ = p̂ + σ² · s(p̂)
+ *   where s(p̂) = ∇ log m(p̂) is the score of the marginal density.
+ *
+ * The score is estimated empirically via calibration tables that track
+ * the additive correction δ = E[θ|p̂] - E[p̂] = hit_rate - avg_pred.
+ * This δ equals σ²·s(p̂) — the full Tweedie correction term.
+ *
+ * Binary tree decomposition: 256-way → 8 binary decisions (MSB to LSB).
+ * Multi-step: K=3 denoising steps with independent score tables.
+ * Calibration context: (step, bit_context, order, shape, confidence, prob_bin)
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <math.h>
+#include "fastmath.h"
+
+#define TWD_NSYM 256
+
+/* Number of reverse diffusion steps */
+#define TWD_STEPS 3
+
+/* Binary tree: 8 levels for 256 symbols */
+#define TWD_N_LEVELS 8
+
+/* 255 internal nodes: 1 + 2 + 4 + ... + 128 */
+#define TWD_N_NODES 255
+
+/* Bit context: encodes level + parent bit values. 27 total. */
+#define TWD_N_BCTX 27
+
+/* Calibration dimensions */
+#define TWD_N_ORD    3    /* order groups: {-1,0,1}, {2,3}, {4+} */
+#define TWD_N_SHAPE  4    /* distribution shape bins by max_p */
+#define TWD_N_CONF   8    /* confidence bins (log-spaced) */
+#define TWD_N_PROB  20    /* binary probability bins (logit-spaced) */
+
+/* Smoothing pseudo-observations per bucket */
+#define TWD_PRIOR_WEIGHT 32.0
+
+/* Logit range for binary probability mapping */
+#define TWD_LOGIT_RANGE 8.0
+
+typedef struct {
+    double sum_pred;   /* sum of predicted P(right) */
+    double hits;       /* times true symbol went right */
+    double total;      /* total observations */
+} TwdCalibEntry;
+
+typedef struct {
+    /* Calibration table: [step][bctx][order][shape][conf][prob_bin]
+     * Total entries: 3 × 27 × 3 × 4 × 8 × 20 = 155,520
+     * Memory: 155,520 × 24 bytes = 3.6 MB */
+    TwdCalibEntry table[TWD_STEPS][TWD_N_BCTX][TWD_N_ORD][TWD_N_SHAPE][TWD_N_CONF][TWD_N_PROB];
+
+    /* Cached from denoise, reused by update */
+    double cached_p_right[TWD_STEPS][TWD_N_NODES];
+    int    cached_prob_bin[TWD_STEPS][TWD_N_NODES];
+    int    cached_bctx[TWD_STEPS][TWD_N_NODES];
+    int    cached_ord;
+    int    cached_shape;
+    int    cached_conf;
+} TweedieDenoiser;
+
+/* ── Bucket mapping functions ── */
+
+static inline int twd_order_group(int ppm_order) {
+    if (ppm_order <= 1) return 0;
+    if (ppm_order <= 3) return 1;
+    return 2;
+}
+
+static inline int twd_shape_bin(double max_p) {
+    if (max_p < 0.05) return 0;   /* very flat */
+    if (max_p < 0.15) return 1;   /* moderately flat */
+    if (max_p < 0.40) return 2;   /* moderate peak */
+    return 3;                      /* peaked */
+}
+
+static inline int twd_conf_bin(double confidence) {
+    if (confidence < 4.0) return 0;
+    int bin = (int)(fast_log(confidence) * (1.0 / 1.3862943611198906));
+    if (bin < 0) bin = 0;
+    if (bin > TWD_N_CONF - 1) bin = TWD_N_CONF - 1;
+    return bin;
+}
+
+/* Binary probability bin: logit-spaced in [-8, 8]. */
+static inline int twd_prob_bin(double p) {
+    if (p < 1e-8) p = 1e-8;
+    if (p > 1.0 - 1e-8) p = 1.0 - 1e-8;
+    double logit = fast_log(p / (1.0 - p));
+    int bin = (int)((logit + TWD_LOGIT_RANGE) / (2.0 * TWD_LOGIT_RANGE) * TWD_N_PROB);
+    if (bin < 0) bin = 0;
+    if (bin > TWD_N_PROB - 1) bin = TWD_N_PROB - 1;
+    return bin;
+}
+
+/* Bin center for prior initialization */
+static inline double twd_bin_center(int bin) {
+    double logit = ((bin + 0.5) / TWD_N_PROB) * 2.0 * TWD_LOGIT_RANGE - TWD_LOGIT_RANGE;
+    return 1.0 / (1.0 + fast_exp(-logit));
+}
+
+/* Bit context: maps (level, node_index_at_level) → context ID 0..26. */
+static inline int twd_bit_context(int level, int node_at_level) {
+    if (level == 0) return 0;
+    if (level == 1) return 1 + node_at_level;              /* 2 contexts */
+    if (level == 2) return 3 + node_at_level;              /* 4 contexts */
+    /* Levels 3-7: hash node_at_level into 4 groups */
+    int group = (node_at_level * 2654435761U) >> 30;       /* hash → 0..3 */
+    return 7 + (level - 3) * 4 + group;
+}
+
+/* ── Initialization ── */
+
+static inline void tweedie_init(TweedieDenoiser *td) {
+    memset(td, 0, sizeof(*td));
+
+    for (int t = 0; t < TWD_STEPS; t++)
+        for (int b = 0; b < TWD_N_BCTX; b++)
+            for (int o = 0; o < TWD_N_ORD; o++)
+                for (int s = 0; s < TWD_N_SHAPE; s++)
+                    for (int c = 0; c < TWD_N_CONF; c++)
+                        for (int p = 0; p < TWD_N_PROB; p++) {
+                            double center = twd_bin_center(p);
+                            td->table[t][b][o][s][c][p].sum_pred = center * TWD_PRIOR_WEIGHT;
+                            td->table[t][b][o][s][c][p].hits     = center * TWD_PRIOR_WEIGHT;
+                            td->table[t][b][o][s][c][p].total    = TWD_PRIOR_WEIGHT;
+                        }
+}
+
+/* ── Denoise: multi-step Tweedie reverse diffusion ──
+ *
+ * Additive Tweedie correction: p' = p + δ
+ * where δ = hits/total - sum_pred/total estimates the Tweedie term σ²·s(p̂).
+ *
+ * This is the nonparametric Tweedie estimator: within each calibration bin,
+ * the empirical hit rate is the posterior mean E[θ|p̂], and the additive
+ * correction δ = E[θ|p̂] - E[p̂] equals σ²·∇log m(p̂). */
+
+static inline void tweedie_denoise(TweedieDenoiser *td, double *probs,
+                                     int ppm_order, double confidence) {
+    int og = twd_order_group(ppm_order);
+    int cb = twd_conf_bin(confidence);
+
+    /* Shape from the 256-way distribution (before any correction) */
+    double max_p = 0.0;
+    for (int i = 0; i < TWD_NSYM; i++)
+        if (probs[i] > max_p) max_p = probs[i];
+    int sb = twd_shape_bin(max_p);
+
+    td->cached_ord = og;
+    td->cached_shape = sb;
+    td->cached_conf = cb;
+
+    double stree[512];
+    double scale[512];
+
+    for (int step = 0; step < TWD_STEPS; step++) {
+
+        /* 1. Build sum tree bottom-up */
+        for (int i = 0; i < TWD_NSYM; i++)
+            stree[TWD_NSYM + i] = probs[i];
+        for (int i = TWD_NSYM - 1; i >= 1; i--)
+            stree[i] = stree[2 * i] + stree[2 * i + 1];
+
+        /* 2. Process all nodes: compute P(right), apply Tweedie correction */
+        scale[1] = 1.0;
+
+        for (int level = 0; level < TWD_N_LEVELS; level++) {
+            int level_start = 1 << level;
+            int level_end   = 1 << (level + 1);
+
+            for (int ni = level_start; ni < level_end; ni++) {
+                double node_total = stree[ni];
+                int node_id = ni - 1;
+                int node_at_level = ni - level_start;
+
+                if (node_total < 1e-15) {
+                    scale[2 * ni]     = scale[ni];
+                    scale[2 * ni + 1] = scale[ni];
+                    td->cached_p_right[step][node_id] = 0.5;
+                    td->cached_prob_bin[step][node_id] = twd_prob_bin(0.5);
+                    td->cached_bctx[step][node_id] = twd_bit_context(level, node_at_level);
+                    continue;
+                }
+
+                double sum_right = stree[2 * ni + 1];
+                double p_right = sum_right / node_total;
+                if (p_right < 1e-8) p_right = 1e-8;
+                if (p_right > 1.0 - 1e-8) p_right = 1.0 - 1e-8;
+
+                int bctx = twd_bit_context(level, node_at_level);
+                int pbin = twd_prob_bin(p_right);
+                td->cached_p_right[step][node_id] = p_right;
+                td->cached_prob_bin[step][node_id] = pbin;
+                td->cached_bctx[step][node_id] = bctx;
+
+                /* Tweedie additive correction: δ = E[θ|p̂] - E[p̂] */
+                TwdCalibEntry *e = &td->table[step][bctx][og][sb][cb][pbin];
+                double avg_pred = e->sum_pred / e->total;
+                double emp_rate = e->hits / e->total;
+                double delta = emp_rate - avg_pred;
+
+                double p_right_corr = p_right + delta;
+                if (p_right_corr < 1e-8)       p_right_corr = 1e-8;
+                if (p_right_corr > 1.0 - 1e-8) p_right_corr = 1.0 - 1e-8;
+
+                double sl = (1.0 - p_right_corr) / (1.0 - p_right);
+                double sr = p_right_corr / p_right;
+                scale[2 * ni]     = scale[ni] * sl;
+                scale[2 * ni + 1] = scale[ni] * sr;
+            }
+        }
+
+        /* 3. Apply accumulated leaf scales */
+        for (int i = 0; i < TWD_NSYM; i++)
+            probs[i] *= scale[TWD_NSYM + i];
+
+        /* 4. Renormalize */
+        double sum = 0.0;
+        for (int i = 0; i < TWD_NSYM; i++) {
+            if (probs[i] < 1e-10) probs[i] = 1e-10;
+            sum += probs[i];
+        }
+        double inv = 1.0 / sum;
+        for (int i = 0; i < TWD_NSYM; i++)
+            probs[i] *= inv;
+
+        /* Recompute shape after correction for next step */
+        max_p = 0.0;
+        for (int i = 0; i < TWD_NSYM; i++)
+            if (probs[i] > max_p) max_p = probs[i];
+        sb = twd_shape_bin(max_p);
+    }
+}
+
+/* ── Update ── */
+
+static inline void tweedie_update(TweedieDenoiser *td, uint8_t true_symbol) {
+    int og = td->cached_ord;
+    int sb = td->cached_shape;
+    int cb = td->cached_conf;
+
+    for (int step = 0; step < TWD_STEPS; step++) {
+        for (int level = 0; level < TWD_N_LEVELS; level++) {
+            int block_size = TWD_NSYM >> level;
+            int half       = block_size >> 1;
+
+            int node_at_level = true_symbol / block_size;
+            int start = node_at_level * block_size;
+            int mid   = start + half;
+            int went_right = (true_symbol >= mid) ? 1 : 0;
+
+            int node_id = (1 << level) - 1 + node_at_level;
+            int pbin = td->cached_prob_bin[step][node_id];
+            int bctx = td->cached_bctx[step][node_id];
+
+            TwdCalibEntry *e = &td->table[step][bctx][og][sb][cb][pbin];
+            e->sum_pred += td->cached_p_right[step][node_id];
+            e->total    += 1.0;
+            if (went_right)
+                e->hits += 1.0;
+        }
+    }
+}
+
+#endif /* TWEEDIE_H */

word.h

@@ -0,0 +1,443 @@
+#ifndef WORD_H
+#define WORD_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define WORD_NSYM 256
+
+/* ── Word character set ── */
+static inline int is_word_char(int c) {
+    if (c >= 'a' && c <= 'z') return 1;
+    if (c >= 'A' && c <= 'Z') return 1;
+    if (c >= '0' && c <= '9') return 1;
+    if (c == '\'' || c == '-') return 1;
+    return 0;
+}
+
+/* ── Trie node ── */
+typedef struct TrieNode {
+    /* continuations: next_byte → count */
+    int cont_keys[64];
+    int cont_vals[64];
+    int cont_count;
+
+    /* children: byte → child node */
+    struct TrieNode *children[256];
+} TrieNode;
+
+static inline TrieNode *trie_new(void) {
+    TrieNode *n = (TrieNode *)calloc(1, sizeof(TrieNode));
+    return n;
+}
+
+static inline void trie_free(TrieNode *n) {
+    if (!n) return;
+    for (int i = 0; i < 256; i++)
+        trie_free(n->children[i]);
+    free(n);
+}
+
+static inline void trie_add_cont(TrieNode *n, int byte_val) {
+    for (int i = 0; i < n->cont_count; i++) {
+        if (n->cont_keys[i] == byte_val) {
+            n->cont_vals[i]++;
+            return;
+        }
+    }
+    if (n->cont_count < 64) {
+        n->cont_keys[n->cont_count] = byte_val;
+        n->cont_vals[n->cont_count] = 1;
+        n->cont_count++;
+    }
+}
+
+/* ── Word counts hash table ── */
+typedef struct {
+    uint64_t key;
+    int count;
+} WordCountEntry;
+
+typedef struct {
+    WordCountEntry *entries;
+    uint32_t capacity;
+    uint32_t mask;
+    uint32_t used;
+} WordCountHT;
+
+static inline void wcht_init(WordCountHT *t, uint32_t cap) {
+    t->capacity = cap;
+    t->mask = cap - 1;
+    t->used = 0;
+    t->entries = (WordCountEntry *)calloc(cap, sizeof(WordCountEntry));
+}
+
+static inline void wcht_free(WordCountHT *t) {
+    free(t->entries);
+}
+
+static inline uint64_t word_hash(const uint8_t *w, int len) {
+    uint64_t h = 14695981039346656037ULL;
+    for (int i = 0; i < len; i++) {
+        h ^= w[i];
+        h *= 1099511628211ULL;
+    }
+    if (h == 0) h = 1;
+    return h;
+}
+
+static inline void wcht_grow(WordCountHT *t) {
+    uint32_t old_cap = t->capacity;
+    WordCountEntry *old = t->entries;
+    t->capacity *= 2;
+    t->mask = t->capacity - 1;
+    t->entries = (WordCountEntry *)calloc(t->capacity, sizeof(WordCountEntry));
+    t->used = 0;
+    for (uint32_t i = 0; i < old_cap; i++) {
+        if (old[i].key != 0) {
+            uint32_t idx = (uint32_t)(old[i].key & t->mask);
+            while (t->entries[idx].key != 0)
+                idx = (idx + 1) & t->mask;
+            t->entries[idx] = old[i];
+            t->used++;
+        }
+    }
+    free(old);
+}
+
+static inline int wcht_get(WordCountHT *t, uint64_t key) {
+    uint32_t idx = (uint32_t)(key & t->mask);
+    for (;;) {
+        if (t->entries[idx].key == key) return t->entries[idx].count;
+        if (t->entries[idx].key == 0) return 0;
+        idx = (idx + 1) & t->mask;
+    }
+}
+
+static inline void wcht_add(WordCountHT *t, uint64_t key, int delta) {
+    if (t->used * 5 > t->capacity * 3) wcht_grow(t);
+    uint32_t idx = (uint32_t)(key & t->mask);
+    for (;;) {
+        if (t->entries[idx].key == key) {
+            t->entries[idx].count += delta;
+            return;
+        }
+        if (t->entries[idx].key == 0) {
+            t->entries[idx].key = key;
+            t->entries[idx].count = delta;
+            t->used++;
+            return;
+        }
+        idx = (idx + 1) & t->mask;
+    }
+}
+
+/* ── Bigram table: word_hash → { byte → count } ── */
+typedef struct {
+    uint64_t key;
+    int counts[256];
+    int total;
+} BigramEntry;
+
+typedef struct {
+    BigramEntry *entries;
+    uint32_t capacity;
+    uint32_t mask;
+    uint32_t used;
+} BigramHT;
+
+static inline void bht_init(BigramHT *t, uint32_t cap) {
+    t->capacity = cap;
+    t->mask = cap - 1;
+    t->used = 0;
+    t->entries = (BigramEntry *)calloc(cap, sizeof(BigramEntry));
+}
+
+static inline void bht_free(BigramHT *t) {
+    free(t->entries);
+}
+
+static inline void bht_grow(BigramHT *t) {
+    uint32_t old_cap = t->capacity;
+    BigramEntry *old = t->entries;
+    t->capacity *= 2;
+    t->mask = t->capacity - 1;
+    t->entries = (BigramEntry *)calloc(t->capacity, sizeof(BigramEntry));
+    t->used = 0;
+    for (uint32_t i = 0; i < old_cap; i++) {
+        if (old[i].key != 0) {
+            uint32_t idx = (uint32_t)(old[i].key & t->mask);
+            while (t->entries[idx].key != 0)
+                idx = (idx + 1) & t->mask;
+            t->entries[idx] = old[i];
+            t->used++;
+        }
+    }
+    free(old);
+}
+
+static inline BigramEntry *bht_get_or_create(BigramHT *t, uint64_t key) {
+    if (t->used * 5 > t->capacity * 3) bht_grow(t);
+    uint32_t idx = (uint32_t)(key & t->mask);
+    for (;;) {
+        if (t->entries[idx].key == key) return &t->entries[idx];
+        if (t->entries[idx].key == 0) {
+            t->entries[idx].key = key;
+            memset(t->entries[idx].counts, 0, sizeof(t->entries[idx].counts));
+            t->entries[idx].total = 0;
+            t->used++;
+            return &t->entries[idx];
+        }
+        idx = (idx + 1) & t->mask;
+    }
+}
+
+/* ── Word Model ── */
+
+typedef struct {
+    TrieNode *trie;
+    WordCountHT word_counts;
+    BigramHT bigrams;
+
+    uint8_t current_word[256];
+    int current_word_len;
+
+    uint8_t last_word[256];
+    int last_word_len;
+    int has_last_word;
+
+    int in_word;
+    double hits;
+    double attempts;
+
+    /* prediction cache to avoid double trie traversal */
+    double cached_probs[WORD_NSYM];
+    double cached_conf;
+    int cache_valid;
+} WordModel;
+
+static inline void word_init(WordModel *w) {
+    w->trie = trie_new();
+    wcht_init(&w->word_counts, 4096);
+    bht_init(&w->bigrams, 2048);
+    w->current_word_len = 0;
+    w->last_word_len = 0;
+    w->has_last_word = 0;
+    w->in_word = 0;
+    w->hits = 1.0;
+    w->attempts = 2.0;
+    w->cache_valid = 0;
+}
+
+static inline void word_free(WordModel *w) {
+    trie_free(w->trie);
+    wcht_free(&w->word_counts);
+    bht_free(&w->bigrams);
+}
+
+static inline void word_add_to_trie(WordModel *w, const uint8_t *word, int len) {
+    uint64_t wh = word_hash(word, len);
+    wcht_add(&w->word_counts, wh, 1);
+
+    TrieNode *node = w->trie;
+    for (int i = 0; i < len; i++) {
+        int b = word[i];
+        if (!node->children[b])
+            node->children[b] = trie_new();
+        TrieNode *entry = node->children[b];
+        if (i + 1 < len)
+            trie_add_cont(entry, word[i + 1]);
+        node = entry;
+    }
+}
+
+/* Get continuations for a prefix. Returns count of distinct continuations.
+   Fills keys[] and vals[] arrays. */
+static inline int word_get_continuations(WordModel *w, const uint8_t *prefix,
+                                          int prefix_len, int *keys, int *vals) {
+    if (prefix_len == 0) return 0;
+    TrieNode *node = w->trie;
+    for (int i = 0; i < prefix_len; i++) {
+        int b = prefix[i];
+        if (!node->children[b]) return 0;
+        TrieNode *entry = node->children[b];
+        if (i == prefix_len - 1) {
+            int n = entry->cont_count;
+            for (int j = 0; j < n; j++) {
+                keys[j] = entry->cont_keys[j];
+                vals[j] = entry->cont_vals[j];
+            }
+            return n;
+        }
+        node = entry;
+    }
+    return 0;
+}
+
+/*
+ * predict: fills probs[256] if prediction available.
+ * Returns 1 with confidence in *out_conf, or 0 if no prediction.
+ */
+static inline int word_predict(WordModel *w, double *probs, double *out_conf) {
+    static const int boundary_chars[] = {32, 10, 13, 44, 46, 59, 58, 33, 63, 41, 93};
+    static const int n_boundary = 11;
+
+    if (w->in_word && w->current_word_len >= 1) {
+        int keys[64], vals[64];
+        int nc = word_get_continuations(w, w->current_word,
+                                         w->current_word_len, keys, vals);
+        if (nc > 0) {
+            memset(probs, 0, WORD_NSYM * sizeof(double));
+            int total = 0;
+            for (int i = 0; i < nc; i++) total += vals[i];
+            double inv_total = 1.0 / total;
+            for (int i = 0; i < nc; i++)
+                probs[keys[i]] += vals[i] * inv_total;
+
+            /* word boundary probability */
+            uint64_t wh = word_hash(w->current_word, w->current_word_len);
+            int wcount = wcht_get(&w->word_counts, wh);
+            if (wcount > 0) {
+                double bw = (double)wcount / (wcount + total);
+                for (int i = 0; i < WORD_NSYM; i++)
+                    probs[i] *= (1.0 - bw);
+                for (int i = 0; i < n_boundary; i++)
+                    probs[boundary_chars[i]] += bw / n_boundary;
+            }
+
+            int plen = w->current_word_len;
+            double confidence = (plen / 3.0 < 1.0 ? plen / 3.0 : 1.0);
+            double cont_factor = nc * 0.5;
+            if (cont_factor > 1.0) cont_factor = 1.0;
+            confidence *= cont_factor;
+            confidence *= (w->hits / w->attempts);
+
+            double sum = 0.0;
+            for (int i = 0; i < WORD_NSYM; i++) sum += probs[i];
+            if (sum > 0.0) {
+                double inv = 1.0 / sum;
+                for (int i = 0; i < WORD_NSYM; i++) probs[i] *= inv;
+                *out_conf = confidence;
+                return 1;
+            }
+        }
+    } else if (!w->in_word && w->has_last_word) {
+        uint64_t wh = word_hash(w->last_word, w->last_word_len);
+        BigramEntry *be = NULL;
+        /* look up without creating */
+        uint32_t idx = (uint32_t)(wh & w->bigrams.mask);
+        for (;;) {
+            if (w->bigrams.entries[idx].key == wh) {
+                be = &w->bigrams.entries[idx];
+                break;
+            }
+            if (w->bigrams.entries[idx].key == 0) break;
+            idx = (idx + 1) & w->bigrams.mask;
+        }
+        if (be && be->total > 0) {
+            memset(probs, 0, WORD_NSYM * sizeof(double));
+            double inv = 1.0 / be->total;
+            for (int i = 0; i < WORD_NSYM; i++)
+                if (be->counts[i] > 0)
+                    probs[i] = be->counts[i] * inv;
+
+            double confidence = (be->total / 5.0 < 1.0 ? be->total / 5.0 : 1.0);
+            confidence *= 0.3 * (w->hits / w->attempts);
+
+            double sum = 0.0;
+            for (int i = 0; i < WORD_NSYM; i++) sum += probs[i];
+            if (sum > 0.0) {
+                double inv2 = 1.0 / sum;
+                for (int i = 0; i < WORD_NSYM; i++) probs[i] *= inv2;
+                *out_conf = confidence;
+                return 1;
+            }
+        }
+    }
+
+    *out_conf = 0.0;
+    return 0;
+}
+
+/* Predict with caching: compute once, reuse in update */
+static inline int word_predict_cached(WordModel *w, double *probs, double *out_conf) {
+    if (w->cache_valid) {
+        memcpy(probs, w->cached_probs, sizeof(w->cached_probs));
+        *out_conf = w->cached_conf;
+        return (*out_conf > 0.0) ? 1 : 0;
+    }
+    int ret = word_predict(w, probs, out_conf);
+    if (ret) {
+        memcpy(w->cached_probs, probs, sizeof(w->cached_probs));
+        w->cached_conf = *out_conf;
+    } else {
+        w->cached_conf = 0.0;
+    }
+    w->cache_valid = 1;
+    return ret;
+}
+
+static inline void word_update(WordModel *w, uint8_t byte_val) {
+    /* track accuracy using cached prediction */
+    double pred_conf = w->cached_conf;
+    int has_pred = w->cache_valid && pred_conf > 0.01;
+    if (has_pred) {
+        w->attempts += 1.0;
+        if (w->cached_probs[byte_val] > 0.05)
+            w->hits += 1.0;
+        if (w->attempts > 500.0) {
+            w->hits *= 0.99;
+            w->attempts *= 0.99;
+        }
+    }
+
+    int is_wc = is_word_char(byte_val);
+    if (is_wc) {
+        if (!w->in_word) {
+            w->current_word_len = 0;
+            w->in_word = 1;
+            /* bigram: last_word → first byte of new word */
+            if (w->has_last_word) {
+                uint64_t wh = word_hash(w->last_word, w->last_word_len);
+                BigramEntry *be = bht_get_or_create(&w->bigrams, wh);
+                be->counts[byte_val]++;
+                be->total++;
+            }
+        }
+        if (w->current_word_len < 255)
+            w->current_word[w->current_word_len++] = byte_val;
+    } else {
+        if (w->in_word && w->current_word_len >= 2) {
+            word_add_to_trie(w, w->current_word, w->current_word_len);
+            w->last_word_len = w->current_word_len;
+            memcpy(w->last_word, w->current_word, w->current_word_len);
+            w->has_last_word = 1;
+        } else if (w->in_word) {
+            w->last_word_len = w->current_word_len;
+            memcpy(w->last_word, w->current_word, w->current_word_len);
+            w->has_last_word = 1;
+        }
+        w->in_word = 0;
+        w->current_word_len = 0;
+    }
+    w->cache_valid = 0; /* invalidate cache after state change */
+}
+
+static inline void blend_word_model(double *probs, const double *word_probs,
+                                     double word_confidence) {
+    if (word_confidence < 0.01) return;
+    double weight = word_confidence * 0.35;
+    if (weight > 0.45) weight = 0.45;
+    double sum = 0.0;
+    for (int i = 0; i < WORD_NSYM; i++) {
+        probs[i] = probs[i] * (1.0 - weight) + word_probs[i] * weight;
+        if (probs[i] < 1e-10) probs[i] = 1e-10;
+        sum += probs[i];
+    }
+    double inv = 1.0 / sum;
+    for (int i = 0; i < WORD_NSYM; i++)
+        probs[i] *= inv;
+}
+
+#endif /* WORD_H */