Created
October 7, 2025 00:23
-
-
Save lardratboy/143640242770d06beb0177afb3447c6a to your computer and use it in GitHub Desktop.
tool to export tensor data from safetensors
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>SafeTensor Information Dump</title> | |
| <style> | |
| * { | |
| box-sizing: border-box; | |
| } | |
| body { | |
| font-family: 'Courier New', monospace; | |
| margin: 0; | |
| padding: 20px; | |
| background: #1a1a1a; | |
| color: #e0e0e0; | |
| line-height: 1.4; | |
| } | |
| .container { | |
| max-width: 1400px; | |
| margin: 0 auto; | |
| } | |
| header { | |
| text-align: center; | |
| margin-bottom: 30px; | |
| padding: 20px; | |
| background: #2d2d2d; | |
| border-radius: 8px; | |
| border: 2px solid #444; | |
| } | |
| h1 { | |
| color: #4CAF50; | |
| margin: 0; | |
| font-size: 2em; | |
| } | |
| .file-input-section { | |
| margin-bottom: 30px; | |
| padding: 20px; | |
| background: #2d2d2d; | |
| border-radius: 8px; | |
| border: 2px solid #444; | |
| } | |
| .file-input { | |
| width: 100%; | |
| padding: 10px; | |
| background: #1a1a1a; | |
| color: #e0e0e0; | |
| border: 2px solid #666; | |
| border-radius: 4px; | |
| font-family: inherit; | |
| } | |
| .controls { | |
| display: flex; | |
| gap: 20px; | |
| align-items: center; | |
| margin: 20px 0; | |
| flex-wrap: wrap; | |
| } | |
| .control-group { | |
| display: flex; | |
| align-items: center; | |
| gap: 10px; | |
| } | |
| label { | |
| color: #4CAF50; | |
| font-weight: bold; | |
| } | |
| input[type="number"], select { | |
| padding: 5px; | |
| background: #1a1a1a; | |
| color: #e0e0e0; | |
| border: 1px solid #666; | |
| border-radius: 4px; | |
| font-family: inherit; | |
| } | |
| .info-grid { | |
| display: grid; | |
| grid-template-columns: 1fr 1fr; | |
| gap: 20px; | |
| margin-bottom: 30px; | |
| } | |
| .info-panel { | |
| background: #2d2d2d; | |
| padding: 20px; | |
| border-radius: 8px; | |
| border: 2px solid #444; | |
| } | |
| .info-panel h3 { | |
| color: #4CAF50; | |
| margin-top: 0; | |
| border-bottom: 1px solid #666; | |
| padding-bottom: 10px; | |
| } | |
| .tensor-list { | |
| max-height: 300px; | |
| overflow-y: auto; | |
| background: #1a1a1a; | |
| padding: 10px; | |
| border-radius: 4px; | |
| border: 1px solid #666; | |
| } | |
| .tensor-item { | |
| padding: 8px; | |
| margin: 4px 0; | |
| background: #333; | |
| border-radius: 4px; | |
| cursor: pointer; | |
| transition: background 0.2s; | |
| } | |
| .tensor-item:hover { | |
| background: #444; | |
| } | |
| .tensor-item.selected { | |
| background: #4CAF50; | |
| color: #000; | |
| } | |
| .tensor-name { | |
| font-weight: bold; | |
| color: #4CAF50; | |
| } | |
| .tensor-details { | |
| font-size: 0.9em; | |
| color: #ccc; | |
| margin-top: 4px; | |
| } | |
| .hexdump-container { | |
| background: #2d2d2d; | |
| padding: 20px; | |
| border-radius: 8px; | |
| border: 2px solid #444; | |
| } | |
| .hexdump { | |
| background: #1a1a1a; | |
| padding: 15px; | |
| border-radius: 4px; | |
| border: 1px solid #666; | |
| font-family: 'Courier New', monospace; | |
| font-size: 14px; | |
| white-space: pre; | |
| overflow-x: auto; | |
| max-height: 500px; | |
| overflow-y: auto; | |
| } | |
| .hex-offset { | |
| color: #888; | |
| } | |
| .hex-bytes { | |
| color: #4CAF50; | |
| } | |
| .hex-ascii { | |
| color: #FFA500; | |
| } | |
| .status { | |
| padding: 10px; | |
| margin: 10px 0; | |
| border-radius: 4px; | |
| border-left: 4px solid #4CAF50; | |
| background: #2d2d2d; | |
| } | |
| .error { | |
| border-left-color: #f44336; | |
| background: #3d2d2d; | |
| color: #ffcccb; | |
| } | |
| .export-button { | |
| background: #4CAF50; | |
| color: #000; | |
| border: none; | |
| padding: 10px 20px; | |
| border-radius: 4px; | |
| cursor: pointer; | |
| font-family: inherit; | |
| font-weight: bold; | |
| margin-top: 10px; | |
| transition: background 0.2s; | |
| } | |
| .export-button:hover { | |
| background: #45a049; | |
| } | |
| .export-button:disabled { | |
| background: #666; | |
| color: #999; | |
| cursor: not-allowed; | |
| } | |
| @media (max-width: 768px) { | |
| .info-grid { | |
| grid-template-columns: 1fr; | |
| } | |
| .controls { | |
| flex-direction: column; | |
| align-items: stretch; | |
| } | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="container"> | |
| <header> | |
| <h1>🛡️ SafeTensor Information Dump</h1> | |
| <p>Analyze tensor files with dimensions, metadata, and hexdump visualization</p> | |
| </header> | |
| <div class="file-input-section"> | |
| <input type="file" id="fileInput" class="file-input" accept=".safetensors,.bin,.pt,.pth"> | |
| <div class="controls"> | |
| <div class="control-group"> | |
| <label for="hexdumpSize">Hexdump Size:</label> | |
| <input type="number" id="hexdumpSize" min="64" max="4096" value="512" step="64"> | |
| <span>bytes</span> | |
| </div> | |
| <div class="control-group"> | |
| <label for="hexdumpOffset">Offset:</label> | |
| <input type="number" id="hexdumpOffset" min="0" value="0" step="16"> | |
| </div> | |
| <div class="control-group"> | |
| <label for="bytesPerLine">Bytes per line:</label> | |
| <select id="bytesPerLine"> | |
| <option value="8">8</option> | |
| <option value="16" selected>16</option> | |
| <option value="32">32</option> | |
| </select> | |
| </div> | |
| </div> | |
| </div> | |
| <div id="statusContainer"></div> | |
| <div class="info-grid" id="infoGrid" style="display: none;"> | |
| <div class="info-panel"> | |
| <h3>📋 File Information</h3> | |
| <div id="fileInfo"></div> | |
| </div> | |
| <div class="info-panel"> | |
| <h3>🧮 Tensor Metadata</h3> | |
| <div id="tensorMetadata"></div> | |
| </div> | |
| </div> | |
| <div class="info-grid" id="tensorGrid" style="display: none;"> | |
| <div class="info-panel"> | |
| <h3>📦 Tensors</h3> | |
| <div class="tensor-list" id="tensorList"></div> | |
| </div> | |
| <div class="info-panel"> | |
| <h3>🔍 Selected Tensor Details</h3> | |
| <div id="selectedTensorInfo">Select a tensor to view details</div> | |
| <button id="exportButton" class="export-button" style="display: none;" disabled> | |
| 💾 Export Tensor as .bin | |
| </button> | |
| </div> | |
| </div> | |
| <div class="hexdump-container" id="hexdumpContainer" style="display: none;"> | |
| <h3>🔢 Hexdump</h3> | |
| <div class="hexdump" id="hexdump"></div> | |
| </div> | |
| </div> | |
| <script> | |
| class TensorDump { | |
| constructor() { | |
| this.fileData = null; | |
| this.metadata = null; | |
| this.selectedTensor = null; | |
| this.initializeEventListeners(); | |
| } | |
| initializeEventListeners() { | |
| document.getElementById('fileInput').addEventListener('change', (e) => { | |
| this.handleFileInput(e); | |
| }); | |
| document.getElementById('hexdumpSize').addEventListener('input', () => { | |
| this.updateHexdump(); | |
| }); | |
| document.getElementById('hexdumpOffset').addEventListener('input', () => { | |
| this.updateHexdump(); | |
| }); | |
| document.getElementById('bytesPerLine').addEventListener('change', () => { | |
| this.updateHexdump(); | |
| }); | |
| document.getElementById('exportButton').addEventListener('click', () => { | |
| this.exportSelectedTensor(); | |
| }); | |
| } | |
| showStatus(message, isError = false) { | |
| const container = document.getElementById('statusContainer'); | |
| container.innerHTML = `<div class="status ${isError ? 'error' : ''}">${message}</div>`; | |
| } | |
| async handleFileInput(event) { | |
| const file = event.target.files[0]; | |
| if (!file) return; | |
| this.showStatus('📖 Reading file...'); | |
| try { | |
| const arrayBuffer = await file.arrayBuffer(); | |
| this.fileData = new Uint8Array(arrayBuffer); | |
| this.showStatus('🔍 Analyzing file structure...'); | |
| await this.analyzeFile(file); | |
| } catch (error) { | |
| console.error('Error reading file:', error); | |
| this.showStatus('❌ Error reading file: ' + error.message, true); | |
| } | |
| } | |
| async analyzeFile(file) { | |
| try { | |
| // Try to parse as SafeTensors first | |
| if (await this.parseSafeTensors()) { | |
| this.showStatus('✅ SafeTensors file parsed successfully'); | |
| } else { | |
| // Fall back to generic binary analysis | |
| this.parseGenericBinary(file); | |
| this.showStatus('✅ Binary file loaded (generic format)'); | |
| } | |
| this.displayFileInfo(file); | |
| this.updateHexdump(); | |
| } catch (error) { | |
| console.error('Error analyzing file:', error); | |
| this.showStatus('❌ Error analyzing file: ' + error.message, true); | |
| } | |
| } | |
| async parseSafeTensors() { | |
| if (this.fileData.length < 8) return false; | |
| try { | |
| // Read header length (first 8 bytes, little-endian) | |
| const headerLength = new DataView(this.fileData.buffer).getBigUint64(0, true); | |
| if (headerLength > this.fileData.length - 8) return false; | |
| // Read JSON header | |
| const headerBytes = this.fileData.slice(8, 8 + Number(headerLength)); | |
| const headerText = new TextDecoder().decode(headerBytes); | |
| this.metadata = JSON.parse(headerText); | |
| // Calculate data offset | |
| this.dataOffset = 8 + Number(headerLength); | |
| return true; | |
| } catch (error) { | |
| console.error('Failed to parse as SafeTensors:', error); | |
| return false; | |
| } | |
| } | |
| parseGenericBinary(file) { | |
| this.metadata = { | |
| "__metadata__": { | |
| "format": "Generic Binary", | |
| "file_size": this.fileData.length, | |
| "file_name": file.name | |
| } | |
| }; | |
| this.dataOffset = 0; | |
| } | |
| displayFileInfo(file) { | |
| const fileInfo = document.getElementById('fileInfo'); | |
| const tensorMetadata = document.getElementById('tensorMetadata'); | |
| const fileSizeKB = (file.size / 1024).toFixed(2); | |
| const fileSizeMB = (file.size / (1024 * 1024)).toFixed(2); | |
| fileInfo.innerHTML = ` | |
| <div><strong>Name:</strong> ${file.name}</div> | |
| <div><strong>Size:</strong> ${fileSizeKB} KB (${fileSizeMB} MB)</div> | |
| <div><strong>Type:</strong> ${file.type || 'Unknown'}</div> | |
| <div><strong>Last Modified:</strong> ${new Date(file.lastModified).toLocaleString()}</div> | |
| <div><strong>Data Offset:</strong> ${this.dataOffset} bytes</div> | |
| `; | |
| if (this.metadata && this.metadata.__metadata__) { | |
| const meta = this.metadata.__metadata__; | |
| let metaHtml = '<div><strong>Metadata:</strong></div>'; | |
| for (const [key, value] of Object.entries(meta)) { | |
| metaHtml += `<div style="margin-left: 20px;"><strong>${key}:</strong> ${value}</div>`; | |
| } | |
| tensorMetadata.innerHTML = metaHtml; | |
| } else { | |
| tensorMetadata.innerHTML = '<div>No metadata found</div>'; | |
| } | |
| document.getElementById('infoGrid').style.display = 'grid'; | |
| if (this.metadata && Object.keys(this.metadata).some(key => key !== '__metadata__')) { | |
| this.displayTensors(); | |
| } | |
| } | |
| displayTensors() { | |
| const tensorList = document.getElementById('tensorList'); | |
| tensorList.innerHTML = ''; | |
| const tensors = Object.entries(this.metadata).filter(([key]) => key !== '__metadata__'); | |
| if (tensors.length === 0) { | |
| tensorList.innerHTML = '<div>No tensors found</div>'; | |
| return; | |
| } | |
| tensors.forEach(([name, info]) => { | |
| const tensorItem = document.createElement('div'); | |
| tensorItem.className = 'tensor-item'; | |
| const shape = info.shape || 'Unknown shape'; | |
| const dtype = info.dtype || 'Unknown type'; | |
| const offset = info.data_offsets ? `[${info.data_offsets[0]}, ${info.data_offsets[1]})` : 'Unknown offset'; | |
| tensorItem.innerHTML = ` | |
| <div class="tensor-name">${name}</div> | |
| <div class="tensor-details"> | |
| Shape: [${Array.isArray(shape) ? shape.join(', ') : shape}]<br> | |
| Type: ${dtype}<br> | |
| Offset: ${offset} | |
| </div> | |
| `; | |
| tensorItem.addEventListener('click', () => { | |
| this.selectTensor(name, info, tensorItem); | |
| }); | |
| tensorList.appendChild(tensorItem); | |
| }); | |
| document.getElementById('tensorGrid').style.display = 'grid'; | |
| } | |
| selectTensor(name, info, element) { | |
| // Remove previous selection | |
| document.querySelectorAll('.tensor-item').forEach(item => { | |
| item.classList.remove('selected'); | |
| }); | |
| // Add selection to current item | |
| element.classList.add('selected'); | |
| this.selectedTensor = { name, info }; | |
| const selectedInfo = document.getElementById('selectedTensorInfo'); | |
| let infoHtml = `<div><strong>Name:</strong> ${name}</div>`; | |
| if (info.shape) { | |
| const totalElements = Array.isArray(info.shape) ? | |
| info.shape.reduce((a, b) => a * b, 1) : 'Unknown'; | |
| infoHtml += `<div><strong>Shape:</strong> [${Array.isArray(info.shape) ? info.shape.join(', ') : info.shape}]</div>`; | |
| infoHtml += `<div><strong>Total Elements:</strong> ${totalElements}</div>`; | |
| } | |
| if (info.dtype) { | |
| infoHtml += `<div><strong>Data Type:</strong> ${info.dtype}</div>`; | |
| } | |
| if (info.data_offsets) { | |
| const size = info.data_offsets[1] - info.data_offsets[0]; | |
| infoHtml += `<div><strong>Data Range:</strong> ${info.data_offsets[0]} - ${info.data_offsets[1]} (${size} bytes)</div>`; | |
| // Update hexdump offset to tensor start | |
| document.getElementById('hexdumpOffset').value = this.dataOffset + info.data_offsets[0]; | |
| this.updateHexdump(); | |
| } | |
| selectedInfo.innerHTML = infoHtml; | |
| // Show and enable export button if tensor has valid data offsets | |
| const exportButton = document.getElementById('exportButton'); | |
| if (info.data_offsets && Array.isArray(info.data_offsets) && info.data_offsets.length >= 2) { | |
| exportButton.style.display = 'block'; | |
| exportButton.disabled = false; | |
| } else { | |
| exportButton.style.display = 'none'; | |
| exportButton.disabled = true; | |
| } | |
| } | |
| exportSelectedTensor() { | |
| if (!this.selectedTensor || !this.fileData) { | |
| this.showStatus('❌ No tensor selected or no file loaded', true); | |
| return; | |
| } | |
| const { name, info } = this.selectedTensor; | |
| if (!info.data_offsets || !Array.isArray(info.data_offsets) || info.data_offsets.length < 2) { | |
| this.showStatus('❌ Selected tensor has no valid data offsets', true); | |
| return; | |
| } | |
| try { | |
| // Extract tensor data | |
| const startOffset = this.dataOffset + info.data_offsets[0]; | |
| const endOffset = this.dataOffset + info.data_offsets[1]; | |
| const tensorData = this.fileData.slice(startOffset, endOffset); | |
| // Create blob and download | |
| const blob = new Blob([tensorData], { type: 'application/octet-stream' }); | |
| const url = URL.createObjectURL(blob); | |
| // Create temporary download link | |
| const a = document.createElement('a'); | |
| a.href = url; | |
| a.download = `${name}.bin`; | |
| document.body.appendChild(a); | |
| a.click(); | |
| document.body.removeChild(a); | |
| // Clean up URL | |
| URL.revokeObjectURL(url); | |
| const sizeKB = (tensorData.length / 1024).toFixed(2); | |
| this.showStatus(`✅ Exported "${name}" (${sizeKB} KB) as ${name}.bin`); | |
| } catch (error) { | |
| console.error('Error exporting tensor:', error); | |
| this.showStatus('❌ Error exporting tensor: ' + error.message, true); | |
| } | |
| } | |
| updateHexdump() { | |
| if (!this.fileData) return; | |
| const size = parseInt(document.getElementById('hexdumpSize').value); | |
| const offset = parseInt(document.getElementById('hexdumpOffset').value); | |
| const bytesPerLine = parseInt(document.getElementById('bytesPerLine').value); | |
| const hexdump = this.generateHexdump(this.fileData, offset, size, bytesPerLine); | |
| document.getElementById('hexdump').textContent = hexdump; | |
| document.getElementById('hexdumpContainer').style.display = 'block'; | |
| } | |
| generateHexdump(data, offset, size, bytesPerLine) { | |
| const end = Math.min(offset + size, data.length); | |
| let result = ''; | |
| for (let i = offset; i < end; i += bytesPerLine) { | |
| // Offset | |
| const addr = i.toString(16).padStart(8, '0').toUpperCase(); | |
| result += addr + ' '; | |
| // Hex bytes | |
| let hexLine = ''; | |
| let asciiLine = ''; | |
| for (let j = 0; j < bytesPerLine; j++) { | |
| if (i + j < end) { | |
| const byte = data[i + j]; | |
| hexLine += byte.toString(16).padStart(2, '0').toUpperCase() + ' '; | |
| asciiLine += (byte >= 32 && byte <= 126) ? String.fromCharCode(byte) : '.'; | |
| } else { | |
| hexLine += ' '; | |
| asciiLine += ' '; | |
| } | |
| // Add extra space in the middle for 16-byte lines | |
| if (bytesPerLine === 16 && j === 7) { | |
| hexLine += ' '; | |
| } | |
| } | |
| result += hexLine + ' |' + asciiLine + '|\n'; | |
| } | |
| return result; | |
| } | |
| } | |
| // Initialize the application | |
| document.addEventListener('DOMContentLoaded', () => { | |
| new TensorDump(); | |
| }); | |
| </script> | |
| </body> | |
| </html> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment