jon.recoil.org
this repo has no description
tessera-geotessera: implement dequantization
Add dequantization test with numpy fixtures (2x3x4 int8 embeddings,
2x3 float32 scales). Verifies correct int8 * scale multiplication
and output matrix dimensions.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
tessera-geotessera/test/fixtures/emb_2x3x4.npy
tessera-geotessera/test/fixtures/emb_2x3x4.npy
This is a binary file and will not be displayed.
tessera-geotessera/test/fixtures/scales_2x3.npy
tessera-geotessera/test/fixtures/scales_2x3.npy
This is a binary file and will not be displayed.
+22
tessera-geotessera/test/test_geotessera.ml
+22
tessera-geotessera/test/test_geotessera.ml
···
68
68
"https://dl2.geotessera.org/v1/global_0.1_degree_representation/2021/grid_0.15_52.05/grid_0.15_52.05_scales.npy"
69
69
url);
70
70
]);
71
+
72
+
("dequantize", [
73
+
Alcotest.test_case "2x3x4 fixture" `Quick (fun () ->
74
+
let read f = In_channel.with_open_bin f In_channel.input_all in
75
+
let emb_npy = Npy.of_string (read "fixtures/emb_2x3x4.npy") |> Result.get_ok in
76
+
let scales_npy = Npy.of_string (read "fixtures/scales_2x3.npy") |> Result.get_ok in
77
+
let mat = Geotessera.dequantize ~embeddings:emb_npy ~scales:scales_npy in
78
+
(* Output should be 6 rows (2*3), 4 cols *)
79
+
Alcotest.(check int) "rows" 6 mat.Linalg.rows;
80
+
Alcotest.(check int) "cols" 4 mat.Linalg.cols;
81
+
(* Pixel 0: [1,2,3,4] * 0.1 = [0.1, 0.2, 0.3, 0.4] *)
82
+
let eps = 1e-5 in
83
+
Alcotest.(check (float eps)) "p0_f0" 0.1 (Linalg.mat_get mat 0 0);
84
+
Alcotest.(check (float eps)) "p0_f1" 0.2 (Linalg.mat_get mat 0 1);
85
+
Alcotest.(check (float eps)) "p0_f2" 0.3 (Linalg.mat_get mat 0 2);
86
+
Alcotest.(check (float eps)) "p0_f3" 0.4 (Linalg.mat_get mat 0 3);
87
+
(* Pixel 1: [5,6,7,8] * 0.2 = [1.0, 1.2, 1.4, 1.6] *)
88
+
Alcotest.(check (float eps)) "p1_f0" 1.0 (Linalg.mat_get mat 1 0);
89
+
Alcotest.(check (float eps)) "p1_f1" 1.2 (Linalg.mat_get mat 1 1);
90
+
Alcotest.(check (float eps)) "p1_f2" 1.4 (Linalg.mat_get mat 1 2);
91
+
Alcotest.(check (float eps)) "p1_f3" 1.6 (Linalg.mat_get mat 1 3));
92
+
]);
71
93
]