feat(core): add contextual/strided intra-layer checkpoint digest

Author: MinsungMJ

srcs/core/checkpoint_digest.c

@@ -1,5 +1,7 @@
 #include "lis/checkpoint_digest.h"
+#include "lis/intra_layer_trace.h"
 
+#include <float.h>
 #include <stdint.h>
 #include <string.h>
 
@@ -204,6 +206,354 @@ static void lis_checkpoint_digest_update_tag(lis_sha256_context *context,
     lis_sha256_update(context, (const unsigned char *)tag, length);
 }
 
+#if FLT_RADIX == 2 && FLT_MANT_DIG == 24 && FLT_MAX_EXP == 128
+#define LIS_HAS_IEEE754_BINARY32_FLOAT 1
+#else
+#define LIS_HAS_IEEE754_BINARY32_FLOAT 0
+#endif
+
+static int lis_checkpoint_digest_add_u64(uint64_t *total, uint64_t amount)
+{
+    if (*total > UINT64_MAX - amount) {
+        return 0;
+    }
+    *total += amount;
+    return 1;
+}
+
+static int lis_checkpoint_digest_mul_u64(uint64_t left, uint64_t right,
+                                         uint64_t *product)
+{
+    if (left != 0U && right > UINT64_MAX / left) {
+        return 0;
+    }
+    *product = left * right;
+    return 1;
+}
+
+/*
+ * The record owns a fixed-size C string. Validate both its termination and its
+ * UTF-8 byte grammar before using strlen-style operations in the digest path.
+ */
+static lis_status lis_checkpoint_digest_validate_utf8_identifier(
+    const char *text, size_t capacity, size_t *out_length)
+{
+    size_t length = 0U;
+    size_t index = 0U;
+
+    if (text == NULL || capacity == 0U || out_length == NULL) {
+        return LIS_STATUS_INVALID_ARGUMENT;
+    }
+    while (length < capacity && text[length] != '\0') {
+        ++length;
+    }
+    if (length == 0U) {
+        return LIS_STATUS_INVALID_ARGUMENT;
+    }
+    if (length == capacity) {
+        return LIS_STATUS_FORMAT;
+    }
+
+    while (index < length) {
+        const unsigned char lead = (unsigned char)text[index];
+        size_t continuation_count;
+        size_t continuation_index;
+
+        if (lead < 0x80U) {
+            if (lead < 0x20U || lead == 0x7fU) {
+                return LIS_STATUS_INVALID_ARGUMENT;
+            }
+            ++index;
+            continue;
+        }
+        if (lead >= 0xc2U && lead <= 0xdfU) {
+            continuation_count = 1U;
+        } else if (lead >= 0xe0U && lead <= 0xefU) {
+            continuation_count = 2U;
+        } else if (lead >= 0xf0U && lead <= 0xf4U) {
+            continuation_count = 3U;
+        } else {
+            return LIS_STATUS_FORMAT;
+        }
+        if (continuation_count > length - index - 1U) {
+            return LIS_STATUS_FORMAT;
+        }
+        for (continuation_index = 1U;
+             continuation_index <= continuation_count;
+             ++continuation_index) {
+            const unsigned char byte =
+                (unsigned char)text[index + continuation_index];
+
+            if (byte < 0x80U || byte > 0xbfU) {
+                return LIS_STATUS_FORMAT;
+            }
+        }
+        if ((lead == 0xe0U &&
+             (unsigned char)text[index + 1U] < 0xa0U) ||
+            (lead == 0xedU &&
+             (unsigned char)text[index + 1U] > 0x9fU) ||
+            (lead == 0xf0U &&
+             (unsigned char)text[index + 1U] < 0x90U) ||
+            (lead == 0xf4U &&
+             (unsigned char)text[index + 1U] > 0x8fU)) {
+            return LIS_STATUS_FORMAT;
+        }
+        index += continuation_count + 1U;
+    }
+    *out_length = length;
+    return LIS_STATUS_OK;
+}
+
+static lis_status lis_intra_layer_digest_validate_inputs(
+    const lis_intra_layer_trace_record *record,
+    const lis_intra_layer_observation *observation,
+    const lis_intra_layer_fp32_view *view,
+    const lis_intra_layer_stage_info **out_stage_info,
+    size_t *out_precision_length)
+{
+    const lis_intra_layer_stage_info *stage_info;
+    lis_status status;
+    uint64_t shape_product = UINT64_C(1);
+    size_t index;
+
+    if (record == NULL || observation == NULL || view == NULL ||
+        out_stage_info == NULL || out_precision_length == NULL) {
+        return LIS_STATUS_INVALID_ARGUMENT;
+    }
+    if (!LIS_HAS_IEEE754_BINARY32_FLOAT || sizeof(float) != 4U) {
+        return LIS_STATUS_UNSUPPORTED_DTYPE;
+    }
+    if (record->state != LIS_INTRA_LAYER_RECORD_ACTIVE) {
+        return LIS_STATUS_BAD_STATE;
+    }
+    if (record->runtime_checkpoint_step == 0U ||
+        record->total_layer_count == 0U ||
+        record->target_layer >= record->total_layer_count) {
+        return LIS_STATUS_INVALID_ARGUMENT;
+    }
+    status = lis_checkpoint_digest_validate_utf8_identifier(
+        record->precision_path, sizeof(record->precision_path),
+        out_precision_length);
+    if (status != LIS_STATUS_OK) {
+        return status;
+    }
+
+    stage_info = lis_intra_layer_stage_lookup((size_t)observation->stage);
+    if (stage_info == NULL) {
+        return LIS_STATUS_INVALID_ARGUMENT;
+    }
+    if (observation->phase != LIS_INTRA_LAYER_PHASE_DECODE) {
+        return LIS_STATUS_UNSUPPORTED;
+    }
+    if (observation->runtime_checkpoint_step !=
+            record->runtime_checkpoint_step ||
+        observation->layer_index != record->target_layer ||
+        observation->token_position != record->token_position ||
+        observation->batch_index != 0U ||
+        observation->sequence_index != 0U ||
+        observation->stage_order != stage_info->stage_order ||
+        observation->execution_ordinal != observation->stage_order) {
+        return LIS_STATUS_INVALID_ARGUMENT;
+    }
+    if (observation->rank == 0U ||
+        observation->rank > LIS_INTRA_LAYER_MAX_RANK) {
+        return LIS_STATUS_UNSUPPORTED_SHAPE;
+    }
+    for (index = 0U; index < observation->rank; ++index) {
+        uint64_t dimension = (uint64_t)observation->shape[index];
+
+        if (dimension == 0U) {
+            return LIS_STATUS_UNSUPPORTED_SHAPE;
+        }
+        if (!lis_checkpoint_digest_mul_u64(shape_product, dimension,
+                                           &shape_product)) {
+            return LIS_STATUS_OVERFLOW;
+        }
+    }
+    if (observation->element_count == 0U ||
+        shape_product != (uint64_t)observation->element_count) {
+        return LIS_STATUS_SHAPE_MISMATCH;
+    }
+
+    status = lis_intra_layer_fp32_view_validate(view);
+    if (status != LIS_STATUS_OK) {
+        return status;
+    }
+    if (view->rank != observation->rank ||
+        view->logical_element_count != observation->element_count) {
+        return LIS_STATUS_SHAPE_MISMATCH;
+    }
+    for (index = 0U; index < observation->rank; ++index) {
+        if (view->shape[index] != observation->shape[index]) {
+            return LIS_STATUS_SHAPE_MISMATCH;
+        }
+    }
+    *out_stage_info = stage_info;
+    return LIS_STATUS_OK;
+}
+
+static lis_status lis_intra_layer_digest_preflight_stream_size(
+    const lis_intra_layer_observation *observation,
+    const lis_intra_layer_stage_info *stage_info,
+    size_t precision_length)
+{
+    static const char *const framed_constants[] = {
+        LIS_INTRA_LAYER_DIGEST_VERSION,
+        LIS_INTRA_LAYER_LAYOUT_NAME,
+        LIS_INTRA_LAYER_STAGE_TAXONOMY,
+        LIS_INTRA_LAYER_MODEL_FAMILY,
+        LIS_INTRA_LAYER_PHASE_DECODE_NAME,
+        LIS_CHECKPOINT_DIGEST_OBSERVED_DTYPE,
+        LIS_CHECKPOINT_DIGEST_BYTE_ORDER
+    };
+    uint64_t total = (uint64_t)strlen(LIS_INTRA_LAYER_DIGEST_DOMAIN_TAG) + 1U;
+    uint64_t tensor_bytes;
+    size_t index;
+
+    for (index = 0U;
+         index < sizeof(framed_constants) / sizeof(framed_constants[0]);
+         ++index) {
+        if (!lis_checkpoint_digest_add_u64(
+                &total, UINT64_C(8) +
+                            (uint64_t)strlen(framed_constants[index]))) {
+            return LIS_STATUS_OVERFLOW;
+        }
+    }
+    if (!lis_checkpoint_digest_add_u64(
+            &total, UINT64_C(8) + (uint64_t)precision_length) ||
+        !lis_checkpoint_digest_add_u64(
+            &total, UINT64_C(8) + (uint64_t)strlen(stage_info->stage_id)) ||
+        !lis_checkpoint_digest_add_u64(
+            &total,
+            UINT64_C(8) + (uint64_t)strlen(stage_info->tensor_role)) ||
+        !lis_checkpoint_digest_add_u64(
+            &total,
+            (UINT64_C(10) + (uint64_t)observation->rank) * UINT64_C(8)) ||
+        !lis_checkpoint_digest_mul_u64(
+            (uint64_t)observation->element_count, UINT64_C(4),
+            &tensor_bytes) ||
+        !lis_checkpoint_digest_add_u64(&total, tensor_bytes)) {
+        return LIS_STATUS_OVERFLOW;
+    }
+    if (total > UINT64_MAX / UINT64_C(8)) {
+        return LIS_STATUS_OVERFLOW;
+    }
+    return LIS_STATUS_OK;
+}
+
+lis_status lis_intra_layer_checkpoint_digest_fp32(
+    const lis_intra_layer_trace_record *record,
+    const lis_intra_layer_observation *observation,
+    const lis_intra_layer_fp32_view *view,
+    lis_checkpoint_digest *out)
+{
+    const lis_intra_layer_stage_info *stage_info;
+    lis_sha256_context context;
+    lis_status status;
+    size_t precision_length = 0U;
+    size_t logical_indices[LIS_INTRA_LAYER_MAX_RANK] = {0U};
+    size_t logical_index;
+    unsigned char zero = 0U;
+
+    if (out == NULL) {
+        return LIS_STATUS_INVALID_ARGUMENT;
+    }
+    memset(out, 0, sizeof(*out));
+    status = lis_intra_layer_digest_validate_inputs(
+        record, observation, view, &stage_info, &precision_length);
+    if (status != LIS_STATUS_OK) {
+        return status;
+    }
+    status = lis_intra_layer_digest_preflight_stream_size(
+        observation, stage_info, precision_length);
+    if (status != LIS_STATUS_OK) {
+        return status;
+    }
+
+    lis_sha256_init(&context);
+    lis_sha256_update(
+        &context, (const unsigned char *)LIS_INTRA_LAYER_DIGEST_DOMAIN_TAG,
+        strlen(LIS_INTRA_LAYER_DIGEST_DOMAIN_TAG));
+    lis_sha256_update(&context, &zero, 1U);
+    lis_checkpoint_digest_update_tag(&context,
+                                     LIS_INTRA_LAYER_DIGEST_VERSION);
+    lis_checkpoint_digest_update_tag(&context, LIS_INTRA_LAYER_LAYOUT_NAME);
+    lis_checkpoint_digest_update_u64_le(&context,
+                                        LIS_INTRA_LAYER_LAYOUT_VERSION);
+    lis_checkpoint_digest_update_tag(&context,
+                                     LIS_INTRA_LAYER_STAGE_TAXONOMY);
+    lis_checkpoint_digest_update_tag(&context, LIS_INTRA_LAYER_MODEL_FAMILY);
+    lis_checkpoint_digest_update_tag(&context, record->precision_path);
+    lis_checkpoint_digest_update_tag(&context,
+                                     LIS_INTRA_LAYER_PHASE_DECODE_NAME);
+    lis_checkpoint_digest_update_u64_le(
+        &context, (uint64_t)observation->runtime_checkpoint_step);
+    lis_checkpoint_digest_update_u64_le(
+        &context, (uint64_t)observation->layer_index);
+    lis_checkpoint_digest_update_tag(&context, stage_info->stage_id);
+    lis_checkpoint_digest_update_tag(&context, stage_info->tensor_role);
+    lis_checkpoint_digest_update_u64_le(
+        &context, (uint64_t)observation->batch_index);
+    lis_checkpoint_digest_update_u64_le(
+        &context, (uint64_t)observation->sequence_index);
+    lis_checkpoint_digest_update_u64_le(
+        &context, (uint64_t)observation->token_position);
+    lis_checkpoint_digest_update_u64_le(
+        &context, (uint64_t)observation->stage_order);
+    lis_checkpoint_digest_update_u64_le(
+        &context, (uint64_t)observation->execution_ordinal);
+    lis_checkpoint_digest_update_u64_le(&context,
+                                        (uint64_t)observation->rank);
+    for (logical_index = 0U; logical_index < observation->rank;
+         ++logical_index) {
+        lis_checkpoint_digest_update_u64_le(
+            &context, (uint64_t)observation->shape[logical_index]);
+    }
+    lis_checkpoint_digest_update_tag(
+        &context, LIS_CHECKPOINT_DIGEST_OBSERVED_DTYPE);
+    lis_checkpoint_digest_update_tag(&context,
+                                     LIS_CHECKPOINT_DIGEST_BYTE_ORDER);
+    lis_checkpoint_digest_update_u64_le(
+        &context, (uint64_t)observation->element_count);
+
+    for (logical_index = 0U;
+         logical_index < observation->element_count;
+         ++logical_index) {
+        size_t physical_offset = 0U;
+        size_t dimension;
+        uint32_t bits = 0U;
+        unsigned char bytes[4];
+
+        for (dimension = 0U; dimension < observation->rank; ++dimension) {
+            physical_offset += logical_indices[dimension] *
+                               view->element_strides[dimension];
+        }
+        memcpy(&bits, view->data + physical_offset, sizeof(bits));
+        if ((bits & UINT32_C(0x7f800000)) == UINT32_C(0x7f800000) &&
+            (bits & UINT32_C(0x007fffff)) != 0U) {
+            bits = UINT32_C(0x7fc00000);
+        }
+        bytes[0] = (unsigned char)(bits & UINT32_C(0xff));
+        bytes[1] = (unsigned char)((bits >> 8U) & UINT32_C(0xff));
+        bytes[2] = (unsigned char)((bits >> 16U) & UINT32_C(0xff));
+        bytes[3] = (unsigned char)((bits >> 24U) & UINT32_C(0xff));
+        lis_sha256_update(&context, bytes, sizeof(bytes));
+
+        for (dimension = observation->rank; dimension > 0U; --dimension) {
+            const size_t current = dimension - 1U;
+
+            ++logical_indices[current];
+            if (logical_indices[current] < observation->shape[current]) {
+                break;
+            }
+            logical_indices[current] = 0U;
+        }
+    }
+    lis_sha256_final(&context, out->bytes);
+    out->valid = 1;
+    return LIS_STATUS_OK;
+}
+
 lis_status lis_checkpoint_digest_fp32(
     const char *tensor_role,
     const size_t *shape,

srcs/includes/lis/intra_layer_trace.h

@@ -11,10 +11,10 @@
  * emission of the two additive blocks "intra_layer_checkpoint_layout" and
  * "intra_layer_trace".
  *
- * It carries a caller-supplied checkpoint digest and never computes one: no
- * entry point reads tensor elements, and lis_intra_layer_fp32_view is validated
- * for declared span coherence without ever dereferencing its data pointer. The
- * intra-layer digest stream and its computation are a separate work package.
+ * The record module carries a caller-supplied checkpoint digest and never
+ * computes one while appending or emitting a record. The additive digest entry
+ * point declared below is the sole operation that traverses an FP32 view; the
+ * structural view validator itself never dereferences the data pointer.
  *
  * The include list below is deliberately minimal: it is the mechanism that
  * keeps runtime, loader, and CLI state out of this module.
@@ -198,6 +198,17 @@ typedef struct lis_intra_layer_trace_record {
 _Static_assert(sizeof(lis_intra_layer_trace_record) <= 16384U,
                "intra-layer record must stay a small fixed-size object");
 
+/*
+ * Computes the frozen contextual/strided P4 digest for one observation. The
+ * record must be ACTIVE and the observation coordinate must match its target.
+ * On every failure, a non-NULL output is left invalid and zero-filled.
+ */
+lis_status lis_intra_layer_checkpoint_digest_fp32(
+    const lis_intra_layer_trace_record *record,
+    const lis_intra_layer_observation *observation,
+    const lis_intra_layer_fp32_view *view,
+    lis_checkpoint_digest *out);
+
 /*
  * JSON primitives injected by the caller. The layer-trace writer passes its own
  * existing escaper and %.6g-or-null float writer, so the intra blocks are