Skip to content

Instantly share code, notes, and snippets.

@lardratboy
Created October 7, 2025 00:23
Show Gist options
  • Select an option

  • Save lardratboy/143640242770d06beb0177afb3447c6a to your computer and use it in GitHub Desktop.

Select an option

Save lardratboy/143640242770d06beb0177afb3447c6a to your computer and use it in GitHub Desktop.
tool to export tensor data from safetensors
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>SafeTensor Information Dump</title>
<style>
* {
box-sizing: border-box;
}
body {
font-family: 'Courier New', monospace;
margin: 0;
padding: 20px;
background: #1a1a1a;
color: #e0e0e0;
line-height: 1.4;
}
.container {
max-width: 1400px;
margin: 0 auto;
}
header {
text-align: center;
margin-bottom: 30px;
padding: 20px;
background: #2d2d2d;
border-radius: 8px;
border: 2px solid #444;
}
h1 {
color: #4CAF50;
margin: 0;
font-size: 2em;
}
.file-input-section {
margin-bottom: 30px;
padding: 20px;
background: #2d2d2d;
border-radius: 8px;
border: 2px solid #444;
}
.file-input {
width: 100%;
padding: 10px;
background: #1a1a1a;
color: #e0e0e0;
border: 2px solid #666;
border-radius: 4px;
font-family: inherit;
}
.controls {
display: flex;
gap: 20px;
align-items: center;
margin: 20px 0;
flex-wrap: wrap;
}
.control-group {
display: flex;
align-items: center;
gap: 10px;
}
label {
color: #4CAF50;
font-weight: bold;
}
input[type="number"], select {
padding: 5px;
background: #1a1a1a;
color: #e0e0e0;
border: 1px solid #666;
border-radius: 4px;
font-family: inherit;
}
.info-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 20px;
margin-bottom: 30px;
}
.info-panel {
background: #2d2d2d;
padding: 20px;
border-radius: 8px;
border: 2px solid #444;
}
.info-panel h3 {
color: #4CAF50;
margin-top: 0;
border-bottom: 1px solid #666;
padding-bottom: 10px;
}
.tensor-list {
max-height: 300px;
overflow-y: auto;
background: #1a1a1a;
padding: 10px;
border-radius: 4px;
border: 1px solid #666;
}
.tensor-item {
padding: 8px;
margin: 4px 0;
background: #333;
border-radius: 4px;
cursor: pointer;
transition: background 0.2s;
}
.tensor-item:hover {
background: #444;
}
.tensor-item.selected {
background: #4CAF50;
color: #000;
}
.tensor-name {
font-weight: bold;
color: #4CAF50;
}
.tensor-details {
font-size: 0.9em;
color: #ccc;
margin-top: 4px;
}
.hexdump-container {
background: #2d2d2d;
padding: 20px;
border-radius: 8px;
border: 2px solid #444;
}
.hexdump {
background: #1a1a1a;
padding: 15px;
border-radius: 4px;
border: 1px solid #666;
font-family: 'Courier New', monospace;
font-size: 14px;
white-space: pre;
overflow-x: auto;
max-height: 500px;
overflow-y: auto;
}
.hex-offset {
color: #888;
}
.hex-bytes {
color: #4CAF50;
}
.hex-ascii {
color: #FFA500;
}
.status {
padding: 10px;
margin: 10px 0;
border-radius: 4px;
border-left: 4px solid #4CAF50;
background: #2d2d2d;
}
.error {
border-left-color: #f44336;
background: #3d2d2d;
color: #ffcccb;
}
.export-button {
background: #4CAF50;
color: #000;
border: none;
padding: 10px 20px;
border-radius: 4px;
cursor: pointer;
font-family: inherit;
font-weight: bold;
margin-top: 10px;
transition: background 0.2s;
}
.export-button:hover {
background: #45a049;
}
.export-button:disabled {
background: #666;
color: #999;
cursor: not-allowed;
}
@media (max-width: 768px) {
.info-grid {
grid-template-columns: 1fr;
}
.controls {
flex-direction: column;
align-items: stretch;
}
}
</style>
</head>
<body>
<div class="container">
<header>
<h1>🛡️ SafeTensor Information Dump</h1>
<p>Analyze tensor files with dimensions, metadata, and hexdump visualization</p>
</header>
<div class="file-input-section">
<input type="file" id="fileInput" class="file-input" accept=".safetensors,.bin,.pt,.pth">
<div class="controls">
<div class="control-group">
<label for="hexdumpSize">Hexdump Size:</label>
<input type="number" id="hexdumpSize" min="64" max="4096" value="512" step="64">
<span>bytes</span>
</div>
<div class="control-group">
<label for="hexdumpOffset">Offset:</label>
<input type="number" id="hexdumpOffset" min="0" value="0" step="16">
</div>
<div class="control-group">
<label for="bytesPerLine">Bytes per line:</label>
<select id="bytesPerLine">
<option value="8">8</option>
<option value="16" selected>16</option>
<option value="32">32</option>
</select>
</div>
</div>
</div>
<div id="statusContainer"></div>
<div class="info-grid" id="infoGrid" style="display: none;">
<div class="info-panel">
<h3>📋 File Information</h3>
<div id="fileInfo"></div>
</div>
<div class="info-panel">
<h3>🧮 Tensor Metadata</h3>
<div id="tensorMetadata"></div>
</div>
</div>
<div class="info-grid" id="tensorGrid" style="display: none;">
<div class="info-panel">
<h3>📦 Tensors</h3>
<div class="tensor-list" id="tensorList"></div>
</div>
<div class="info-panel">
<h3>🔍 Selected Tensor Details</h3>
<div id="selectedTensorInfo">Select a tensor to view details</div>
<button id="exportButton" class="export-button" style="display: none;" disabled>
💾 Export Tensor as .bin
</button>
</div>
</div>
<div class="hexdump-container" id="hexdumpContainer" style="display: none;">
<h3>🔢 Hexdump</h3>
<div class="hexdump" id="hexdump"></div>
</div>
</div>
<script>
class TensorDump {
constructor() {
this.fileData = null;
this.metadata = null;
this.selectedTensor = null;
this.initializeEventListeners();
}
initializeEventListeners() {
document.getElementById('fileInput').addEventListener('change', (e) => {
this.handleFileInput(e);
});
document.getElementById('hexdumpSize').addEventListener('input', () => {
this.updateHexdump();
});
document.getElementById('hexdumpOffset').addEventListener('input', () => {
this.updateHexdump();
});
document.getElementById('bytesPerLine').addEventListener('change', () => {
this.updateHexdump();
});
document.getElementById('exportButton').addEventListener('click', () => {
this.exportSelectedTensor();
});
}
showStatus(message, isError = false) {
const container = document.getElementById('statusContainer');
container.innerHTML = `<div class="status ${isError ? 'error' : ''}">${message}</div>`;
}
async handleFileInput(event) {
const file = event.target.files[0];
if (!file) return;
this.showStatus('📖 Reading file...');
try {
const arrayBuffer = await file.arrayBuffer();
this.fileData = new Uint8Array(arrayBuffer);
this.showStatus('🔍 Analyzing file structure...');
await this.analyzeFile(file);
} catch (error) {
console.error('Error reading file:', error);
this.showStatus('❌ Error reading file: ' + error.message, true);
}
}
async analyzeFile(file) {
try {
// Try to parse as SafeTensors first
if (await this.parseSafeTensors()) {
this.showStatus('✅ SafeTensors file parsed successfully');
} else {
// Fall back to generic binary analysis
this.parseGenericBinary(file);
this.showStatus('✅ Binary file loaded (generic format)');
}
this.displayFileInfo(file);
this.updateHexdump();
} catch (error) {
console.error('Error analyzing file:', error);
this.showStatus('❌ Error analyzing file: ' + error.message, true);
}
}
async parseSafeTensors() {
if (this.fileData.length < 8) return false;
try {
// Read header length (first 8 bytes, little-endian)
const headerLength = new DataView(this.fileData.buffer).getBigUint64(0, true);
if (headerLength > this.fileData.length - 8) return false;
// Read JSON header
const headerBytes = this.fileData.slice(8, 8 + Number(headerLength));
const headerText = new TextDecoder().decode(headerBytes);
this.metadata = JSON.parse(headerText);
// Calculate data offset
this.dataOffset = 8 + Number(headerLength);
return true;
} catch (error) {
console.error('Failed to parse as SafeTensors:', error);
return false;
}
}
parseGenericBinary(file) {
this.metadata = {
"__metadata__": {
"format": "Generic Binary",
"file_size": this.fileData.length,
"file_name": file.name
}
};
this.dataOffset = 0;
}
displayFileInfo(file) {
const fileInfo = document.getElementById('fileInfo');
const tensorMetadata = document.getElementById('tensorMetadata');
const fileSizeKB = (file.size / 1024).toFixed(2);
const fileSizeMB = (file.size / (1024 * 1024)).toFixed(2);
fileInfo.innerHTML = `
<div><strong>Name:</strong> ${file.name}</div>
<div><strong>Size:</strong> ${fileSizeKB} KB (${fileSizeMB} MB)</div>
<div><strong>Type:</strong> ${file.type || 'Unknown'}</div>
<div><strong>Last Modified:</strong> ${new Date(file.lastModified).toLocaleString()}</div>
<div><strong>Data Offset:</strong> ${this.dataOffset} bytes</div>
`;
if (this.metadata && this.metadata.__metadata__) {
const meta = this.metadata.__metadata__;
let metaHtml = '<div><strong>Metadata:</strong></div>';
for (const [key, value] of Object.entries(meta)) {
metaHtml += `<div style="margin-left: 20px;"><strong>${key}:</strong> ${value}</div>`;
}
tensorMetadata.innerHTML = metaHtml;
} else {
tensorMetadata.innerHTML = '<div>No metadata found</div>';
}
document.getElementById('infoGrid').style.display = 'grid';
if (this.metadata && Object.keys(this.metadata).some(key => key !== '__metadata__')) {
this.displayTensors();
}
}
displayTensors() {
const tensorList = document.getElementById('tensorList');
tensorList.innerHTML = '';
const tensors = Object.entries(this.metadata).filter(([key]) => key !== '__metadata__');
if (tensors.length === 0) {
tensorList.innerHTML = '<div>No tensors found</div>';
return;
}
tensors.forEach(([name, info]) => {
const tensorItem = document.createElement('div');
tensorItem.className = 'tensor-item';
const shape = info.shape || 'Unknown shape';
const dtype = info.dtype || 'Unknown type';
const offset = info.data_offsets ? `[${info.data_offsets[0]}, ${info.data_offsets[1]})` : 'Unknown offset';
tensorItem.innerHTML = `
<div class="tensor-name">${name}</div>
<div class="tensor-details">
Shape: [${Array.isArray(shape) ? shape.join(', ') : shape}]<br>
Type: ${dtype}<br>
Offset: ${offset}
</div>
`;
tensorItem.addEventListener('click', () => {
this.selectTensor(name, info, tensorItem);
});
tensorList.appendChild(tensorItem);
});
document.getElementById('tensorGrid').style.display = 'grid';
}
selectTensor(name, info, element) {
// Remove previous selection
document.querySelectorAll('.tensor-item').forEach(item => {
item.classList.remove('selected');
});
// Add selection to current item
element.classList.add('selected');
this.selectedTensor = { name, info };
const selectedInfo = document.getElementById('selectedTensorInfo');
let infoHtml = `<div><strong>Name:</strong> ${name}</div>`;
if (info.shape) {
const totalElements = Array.isArray(info.shape) ?
info.shape.reduce((a, b) => a * b, 1) : 'Unknown';
infoHtml += `<div><strong>Shape:</strong> [${Array.isArray(info.shape) ? info.shape.join(', ') : info.shape}]</div>`;
infoHtml += `<div><strong>Total Elements:</strong> ${totalElements}</div>`;
}
if (info.dtype) {
infoHtml += `<div><strong>Data Type:</strong> ${info.dtype}</div>`;
}
if (info.data_offsets) {
const size = info.data_offsets[1] - info.data_offsets[0];
infoHtml += `<div><strong>Data Range:</strong> ${info.data_offsets[0]} - ${info.data_offsets[1]} (${size} bytes)</div>`;
// Update hexdump offset to tensor start
document.getElementById('hexdumpOffset').value = this.dataOffset + info.data_offsets[0];
this.updateHexdump();
}
selectedInfo.innerHTML = infoHtml;
// Show and enable export button if tensor has valid data offsets
const exportButton = document.getElementById('exportButton');
if (info.data_offsets && Array.isArray(info.data_offsets) && info.data_offsets.length >= 2) {
exportButton.style.display = 'block';
exportButton.disabled = false;
} else {
exportButton.style.display = 'none';
exportButton.disabled = true;
}
}
exportSelectedTensor() {
if (!this.selectedTensor || !this.fileData) {
this.showStatus('❌ No tensor selected or no file loaded', true);
return;
}
const { name, info } = this.selectedTensor;
if (!info.data_offsets || !Array.isArray(info.data_offsets) || info.data_offsets.length < 2) {
this.showStatus('❌ Selected tensor has no valid data offsets', true);
return;
}
try {
// Extract tensor data
const startOffset = this.dataOffset + info.data_offsets[0];
const endOffset = this.dataOffset + info.data_offsets[1];
const tensorData = this.fileData.slice(startOffset, endOffset);
// Create blob and download
const blob = new Blob([tensorData], { type: 'application/octet-stream' });
const url = URL.createObjectURL(blob);
// Create temporary download link
const a = document.createElement('a');
a.href = url;
a.download = `${name}.bin`;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
// Clean up URL
URL.revokeObjectURL(url);
const sizeKB = (tensorData.length / 1024).toFixed(2);
this.showStatus(`✅ Exported "${name}" (${sizeKB} KB) as ${name}.bin`);
} catch (error) {
console.error('Error exporting tensor:', error);
this.showStatus('❌ Error exporting tensor: ' + error.message, true);
}
}
updateHexdump() {
if (!this.fileData) return;
const size = parseInt(document.getElementById('hexdumpSize').value);
const offset = parseInt(document.getElementById('hexdumpOffset').value);
const bytesPerLine = parseInt(document.getElementById('bytesPerLine').value);
const hexdump = this.generateHexdump(this.fileData, offset, size, bytesPerLine);
document.getElementById('hexdump').textContent = hexdump;
document.getElementById('hexdumpContainer').style.display = 'block';
}
generateHexdump(data, offset, size, bytesPerLine) {
const end = Math.min(offset + size, data.length);
let result = '';
for (let i = offset; i < end; i += bytesPerLine) {
// Offset
const addr = i.toString(16).padStart(8, '0').toUpperCase();
result += addr + ' ';
// Hex bytes
let hexLine = '';
let asciiLine = '';
for (let j = 0; j < bytesPerLine; j++) {
if (i + j < end) {
const byte = data[i + j];
hexLine += byte.toString(16).padStart(2, '0').toUpperCase() + ' ';
asciiLine += (byte >= 32 && byte <= 126) ? String.fromCharCode(byte) : '.';
} else {
hexLine += ' ';
asciiLine += ' ';
}
// Add extra space in the middle for 16-byte lines
if (bytesPerLine === 16 && j === 7) {
hexLine += ' ';
}
}
result += hexLine + ' |' + asciiLine + '|\n';
}
return result;
}
}
// Initialize the application
document.addEventListener('DOMContentLoaded', () => {
new TensorDump();
});
</script>
</body>
</html>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment