{ "schema": "nodus.public-retained-state-observation.v1", "publication_status": "prepared_public_evidence_not_yet_published", "experiment_date_utc": "2026-11-02", "title": "Retained state across two sequential Nodus eight-H200 nodes", "evidence_type": "selected_field_projection_of_retained_operational_receipts_not_raw_logs_or_independent_attestation", "scope": { "production_platform": true, "nodes": 2, "sequential_replacement": "controlled shutdown of the initial node, then creation of a second node", "samples_per_phase": 1, "gpu_class_per_node": "8 x NVIDIA H200", "public_region": "us-8", "same_native_instance_type": true, "same_native_region": true, "physical_host_identity_equal": "not_established", "retained_filesystem_gib": 512, "checkpoint_test_gpu_count": 1, "checkpoint_model": "torch.nn.Linear(64,16)", "checkpoint_optimizer": "torch.optim.AdamW, learning rate 0.01", "checkpoint_training_batch_shape": [ 128, 64 ], "separate_parallel_gpu_check_ranks": 8, "retained_filesystem_kind": "SharedFilesystem", "native_volume_interface": "File", "guest_mount_fstype": "virtiofs" }, "source_pins": { "platform_commit": "be542b312b5552c3c94e7cf8b7cfc53283d3f811", "payload_sha256": "e5de06a7073e3d920b9daa82bfed52c1e4b521f66a1a59382ba4686c2ad66a3c", "owning_acceptance_report_sha256": "33102328e1afa3090ffa39cdb9a91d95c31eb27c57eb2609a9a5af0ff1e04309", "final_handoff_sha256": "2be35ea022947c7b4ab664c4e5f6a4e3b4c11637154d633f17310e2053bdcaef", "both_job_requests_name_same_payload_sha256": true }, "metric_definitions": { "created_to_platform_ready_seconds": { "start": "Nodus API node created_at", "end": "Nodus API platform_ready_at, after the platform readiness gates", "clock_basis": "platform-recorded UTC timestamps", "excluded": "Pre-creation request and guard setup; later authenticated probe, workload environment preparation and CUDA operation. Platform Ready is not the first-CUDA marker." }, "payload_entry_to_first_completed_cuda_seconds": { "start": "Guest time.time_ns() marker labelled ssh_command_started, emitted at detached payload entry after argument validation", "end": "Guest time.time_ns() marker after torch.zeros(1,device=cuda), add_(1), torch.cuda.synchronize(), and scalar equality check", "clock_basis": "Two wall-clock timestamps from the same guest in each phase; nanoseconds are strings to retain precision. Constant clock offset cancels; no monotonic-clock or clock-adjustment bound was recorded.", "included": "Filesystem identity, eight-H200 inventory and NVLink topology checks; initial-only environment creation/package installation; Python/Torch import and the completed one-element CUDA operation.", "excluded": "Checkpoint restore/advance and eight-rank matmul/NCCL checks, which occur after this marker. Not time to resumed training, full payload completion or earliest possible GPU usability." }, "native_phase_intervals": { "source": "Retained native instance status history", "clock_basis": "Provider-reported UTC timestamps", "limitation": "Observational phases only; STARTING does not separate image boot, storage attachment or platform mount setup." } }, "observations": [ { "phase": "initial", "sample_count": 1, "platform_created_at_utc": "2026-11-02T06:35:06.859659Z", "platform_ready_at_utc": "2026-11-02T06:42:08.06471Z", "created_to_platform_ready_seconds": 421.205051, "payload_entry_at_utc": "2026-11-02T06:42:21.857138+00:00", "payload_entry_epoch_ns": "1793601741857138271", "first_completed_cuda_at_utc": "2026-11-02T06:46:03.511500+00:00", "first_completed_cuda_epoch_ns": "1793601963511500050", "payload_entry_to_first_completed_cuda_seconds": 221.654361779, "native_starting_to_running_seconds": 332.287011, "native_new_to_confirmed_seconds": 53.679916, "payload_returncode": 0, "payload_timed_out": false, "checkpoint": { "checkpoint_pass": true, "phase": "initial", "step": 8, "checkpoint_sha256": "883c8c0b9b5e5ef5f87e8e436cb1056241476e9cae7bb676eecc9743ef950177", "environment_reused": false, "cache_bytes": 67108864, "cache_sha256": "6255ac9aaf9885e0640cdebe67ea58afe2b1247931170278578ac48298426cec", "expected_input_checkpoint_sha256": null }, "separate_gpu_rank_checks": { "ranks": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "nccl_world": 8, "all_reported_correct": true }, "evidence": { "probe_final_sha256": "871e8a88e670eb23a1d221e6289e97d609958ee37d17706f71a5dfd6ddfcea4c", "durable_job_request_sha256": "fe37ccfc67ae91ae265da885b06bd8374c34977bfe78b999630a2a0bb8ff593a", "durable_job_final_sha256": "b42691228134701845736f05e6a62c529015208125a786123d6bcd59a814530c", "raw_output_sha256": "1cc31e655979eb4c449eb68af6664b8aeda793f3d94521f08e4fca368be15e74", "native_timing_sha256": "498fc0bcb21d344ec36cb0414ee2da6f5f143eb4fd302c969a50b5b1cf14ca37" } }, { "phase": "recovery", "sample_count": 1, "platform_created_at_utc": "2026-11-02T06:47:55.2825Z", "platform_ready_at_utc": "2026-11-02T06:54:08.047347Z", "created_to_platform_ready_seconds": 372.764847, "payload_entry_at_utc": "2026-11-02T06:54:22.475413+00:00", "payload_entry_epoch_ns": "1793602462475413109", "first_completed_cuda_at_utc": "2026-11-02T06:54:41.886330+00:00", "first_completed_cuda_epoch_ns": "1793602481886330019", "payload_entry_to_first_completed_cuda_seconds": 19.41091691, "native_starting_to_running_seconds": 332.472025, "native_new_to_confirmed_seconds": 2.373242, "payload_returncode": 0, "payload_timed_out": false, "checkpoint": { "checkpoint_pass": true, "phase": "recovery", "step": 9, "checkpoint_sha256": "48ea7a3f68241b9f65205feba517876ba0d7e1c3d71f8f891f29fbe5c3a31a39", "environment_reused": true, "cache_bytes": 67108864, "cache_sha256": "6255ac9aaf9885e0640cdebe67ea58afe2b1247931170278578ac48298426cec", "expected_input_checkpoint_sha256": "883c8c0b9b5e5ef5f87e8e436cb1056241476e9cae7bb676eecc9743ef950177" }, "separate_gpu_rank_checks": { "ranks": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "nccl_world": 8, "all_reported_correct": true }, "evidence": { "probe_final_sha256": "2ed29464db6d83de25ef628945d45c8d4aa2ae114a3319b571723645ed96e8da", "durable_job_request_sha256": "d8b75dcb435dbdf648b64887b1ffa1379a7372dab0a7d3b8dcd11212ca4cc199", "durable_job_final_sha256": "1488e5c0641bf5e876cf8f96837f2cad573f41f2f51ac8f38d753f0b20708b28", "raw_output_sha256": "5ae174534bbe39860711b16e415900c05776dc8179bacefbbd8ab7cd806e7dc5", "native_timing_sha256": "d29f29df626a285bcde4f120f8db4c1b30a39bff7ec54b96a4f4375fc484b641" } } ], "checkpoint_continuity": { "initial_saved_step": 8, "recovery_loaded_required_step": 8, "recovery_completed_step": 9, "expected_input_sha256": "883c8c0b9b5e5ef5f87e8e436cb1056241476e9cae7bb676eecc9743ef950177", "recovery_input_hash_checked_before_load": true, "state_loaded": [ "model parameters", "AdamW optimizer state", "CPU Torch RNG state", "CUDA Torch RNG states", "Python random state" ], "finite_loss_and_parameter_change_checked": true, "same_67108864_byte_cache_hash_checked": true, "checkpoint_commit": "Temporary file plus fsync, atomic replacement, directory fsync", "not_established": [ "Bitwise equivalence to an uninterrupted training trajectory", "Large-model training recovery", "Eight-GPU distributed training checkpoint restore", "Checkpoint restore duration" ] }, "runtime_comparability": { "initial": "Creates a new virtual environment and requests torch==2.14.0 from https://download.pytorch.org/whl/cu126; initial log identifies CPython 3.12.3.", "recovery": "Executes the retained interpreter directly, has no installation branch, and checks Torch version, CUDA version, Python version and Torch path against retained initial metadata.", "retained_environment_assertion_passed": true, "payload_same": true, "platform_source_same": true, "differences": [ "Distinct VMs and sequential allocation times", "Prepared Python environment and cache retained for recovery; fresh installation on initial node" ], "not_recorded_in_public_projection": [ "Exact runtime environment.json values are not emitted in these retained outputs", "Physical host/GPU UUID equality", "Driver version and immutable full-image digest equality", "Network/package-cache/host-cache equivalence" ] }, "limitations": [ "One controlled sequential pair, not a randomized benchmark or startup distribution.", "The four observed durations do not establish a causal provisioning speedup. Native STARTING-to-RUNNING durations were nearly unchanged while native confirmation delay differed.", "First-CUDA timing precedes checkpoint restore and parallel checks.", "Guest, platform and native-service clocks are separate; cross-clock offsets were not measured.", "No automatic preemption migration, snapshot restore, interruption frequency, availability guarantee, idle-capacity advantage, competitor ranking, GPU partitioning result or kernel/training-performance uplift is established.", "No model-training benefit or production-runtime qualification for any separate kernel environment is implied." ], "release_provenance": { "method": "Copy selected technical fields from the pinned original receipts; compute payload interval by integer subtraction of the two retained guest nanosecond markers.", "omitted": "Organization/account/project identifiers, node/VM/bid/volume/workspace identifiers, IP/SSH material, private local paths, internal operation names, billing records and access lifecycle details.", "raw_records_released": false, "raw_release_reason": "Owning operational receipts contain private operational identifiers and require a separate reviewed export before release. Their SHA-256 hashes are retained here. The full acceptance report is not downloadable from this evidence file.", "probe_summary_framing": "Each probe summary appends one trailing newline to durable raw output; scientific markers and values otherwise match." } }