Skip to content

Instantly share code, notes, and snippets.

@Nepi24
Created August 17, 2024 03:57
Show Gist options
  • Select an option

  • Save Nepi24/fd2842f250ea38c89f2a73a3edeb1f8c to your computer and use it in GitHub Desktop.

Select an option

Save Nepi24/fd2842f250ea38c89f2a73a3edeb1f8c to your computer and use it in GitHub Desktop.
Multitrack Chords
<html>
<head>
<title>Magenta - multitrack chords</title>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
<meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1, user-scalable=yes">
<link id="favicon" rel="icon" href="https://magenta.tensorflow.org/favicon.ico" type="image/x-icon">
<link href="https://fonts.googleapis.com/css?family=IBM+Plex+Mono:400,700" rel="stylesheet">
<script src="https://cdn.jsdelivr.net/npm/@magenta/music@1.16.0"></script>
</head>
<body>
<div class="content">
<div class="preamble">
<h1>Multitrack Chords</h1>
<p class="about">This demo uses <a href="https://magenta.tensorflow.org/multitrack">MusicVAE</a>, a machine learning model that
is able to interpolate between different musical styles. This interpolation can also follow a chord progression. Try it below!
</p>
<div id="status" class="loading">Loading model (25 MB)...</div>
</div>
<div class="bottom" id="controls" disabled>
<div style="text-align: center;">
<button id="play" class="inverted">Play</button>
<button id="download">Save as MIDI</button>
</div>
<div class="horizontal">
<div id="style1">
<h2>Style 1</h2>
<button id="sample1">Random</button>
</div>
<input id="alpha" type="range" min="0" max="5" value="0">
<div id="style2">
<h2>Style 2</h2>
<button id="sample2">Random</button>
</div>
</div>
<p><b>Chord progression: </b>
<span id="chordsContainer" disabled>
<input id="chord1" type="text" value="Dm">
<input id="chord2" type="text" value="F">
<input id="chord3" type="text" value="Am">
<input id="chord4" type="text" value="G">
</span>
<button id="changeChords">Change</button>
</p>
</div>
</div>
<p class="fineprint">
Made with <a href="https://magenta.tensorflow.org">Magenta.js</a>.
Designed by <a href="https://meowni.ca/">Monica Dinculescu</a>.
Uses samples from <a href='https://www.polyphone-soundfonts.com/en/files/27-instrument-sets/256-sgm-v2-01'>SGM</a>
with modifications by <a href="https://sites.google.com/site/soundfonts4u/">John Nebauer</a>.
May work poorly on mobile.
</p>
</body>
</html>

Multitrack Chords

Uses a Magenta.js MusicVAE model to interpolate between two musical "styles" while playing an editable chord progression.

A Pen by Ian Simon on CodePen.

License.

const QPM = 120;
const STEPS_PER_QUARTER = 24;
const Z_DIM = 256;
const HUMANIZE_SECONDS = 0.01;
const tf = mm.tf;
// Set up Multitrack MusicVAE.
const model = new mm.MusicVAE('https://storage.googleapis.com/magentadata/js/checkpoints/music_vae/multitrack_chords');
// Set up an audio player.
const player = initPlayerAndEffects();
// Get UI elements.
const statusDiv = document.getElementById('status');
const changeChordsButton = document.getElementById('changeChords');
const playButton = document.getElementById('play');
const sampleButton1 = document.getElementById('sample1');
const sampleButton2 = document.getElementById('sample2');
const alphaSlider = document.getElementById('alpha');
const saveButton = document.getElementById('download');
const chordsContainer = document.getElementById('chordsContainer');
const chordInputs = [
document.getElementById('chord1'),
document.getElementById('chord2'),
document.getElementById('chord3'),
document.getElementById('chord4')
];
const numSteps = +alphaSlider.max + 1;
const numChords = chordInputs.length;
// Declare style / sequence variables.
var z1, z2;
var chordSeqs;
var progSeqs;
var changingChords = false;
var playing = false;
var chords = chordInputs.map(c => c.value);
sampleButton1.onclick = updateSample1;
sampleButton2.onclick = updateSample2;
playButton.onclick = togglePlaying;
saveButton.onclick = saveSequence;
changeChordsButton.onclick = toggleChangeChords;
chordInputs.forEach(c => c.oninput = chordChanged);
model.initialize()
.then(() => {
setUpdatingState();
setTimeout(() => {
generateSample(z => {
z1 = z;
generateSample(z => {
z2 = z;
generateProgressions(setStoppedState);
});
});
}, 0);
});
// Sample a latent vector.
function generateSample(doneCallback) {
const z = tf.randomNormal([1, Z_DIM]);
z.data().then(zArray => {
z.dispose();
doneCallback(zArray);
});
}
// Randomly adjust note times.
function humanize(s) {
const seq = mm.sequences.clone(s);
seq.notes.forEach((note) => {
let offset = HUMANIZE_SECONDS * (Math.random() - 0.5);
if (seq.notes.startTime + offset < 0) {
offset = -seq.notes.startTime;
}
if (seq.notes.endTime > seq.totalTime) {
offset = seq.totalTime - seq.notes.endTime;
}
seq.notes.startTime += offset;
seq.notes.endTime += offset;
});
return seq;
}
// Construct spherical linear interpolation tensor.
function slerp(z1, z2, n) {
const norm1 = tf.norm(z1);
const norm2 = tf.norm(z2);
const omega = tf.acos(tf.matMul(tf.div(z1, norm1),
tf.div(z2, norm2),
false, true));
const sinOmega = tf.sin(omega);
const t1 = tf.linspace(1, 0, n);
const t2 = tf.linspace(0, 1, n);
const alpha1 = tf.div(tf.sin(tf.mul(t1, omega)), sinOmega).as2D(n, 1);
const alpha2 = tf.div(tf.sin(tf.mul(t2, omega)), sinOmega).as2D(n, 1);
const z = tf.add(tf.mul(alpha1, z1), tf.mul(alpha2, z2));
return z;
}
// Concatenate multiple NoteSequence objects.
function concatenateSequences(seqs) {
const seq = mm.sequences.clone(seqs[0]);
let numSteps = seqs[0].totalQuantizedSteps;
for (let i=1; i<seqs.length; i++) {
const s = mm.sequences.clone(seqs[i]);
s.notes.forEach(note => {
note.quantizedStartStep += numSteps;
note.quantizedEndStep += numSteps;
seq.notes.push(note);
});
numSteps += s.totalQuantizedSteps;
}
seq.totalQuantizedSteps = numSteps;
return seq;
}
// Interpolate the two styles for a single chord.
function interpolateSamples(chord, doneCallback) {
const z1Tensor = tf.tensor2d(z1, [1, Z_DIM]);
const z2Tensor = tf.tensor2d(z2, [1, Z_DIM]);
const zInterp = slerp(z1Tensor, z2Tensor, numSteps);
model.decode(zInterp, undefined, [chord], STEPS_PER_QUARTER)
.then(sequences => doneCallback(sequences));
}
// Generate interpolations for all chords.
function generateInterpolations(chordIndex, result, doneCallback) {
if (chordIndex === numChords) {
doneCallback(result);
} else {
interpolateSamples(chords[chordIndex], seqs => {
for (let i=0; i<numSteps; i++) {
result[i].push(seqs[i]);
}
generateInterpolations(chordIndex + 1, result, doneCallback);
})
}
}
// Generate chord progression for each alpha.
function generateProgressions(doneCallback) {
let temp = [];
for (let i=0; i<numSteps; i++) {
temp.push([]);
}
generateInterpolations(0, temp, seqs => {
chordSeqs = seqs;
concatSeqs = chordSeqs.map(s => concatenateSequences(s));
progSeqs = concatSeqs.map(seq => {
const mergedSeq = mm.sequences.mergeInstruments(seq);
const progSeq = mm.sequences.unquantizeSequence(mergedSeq);
progSeq.ticksPerQuarter = STEPS_PER_QUARTER;
return progSeq;
});
const fullSeq = concatenateSequences(concatSeqs);
const mergedFullSeq = mm.sequences.mergeInstruments(fullSeq);
setLoadingState();
player.loadSamples(mergedFullSeq)
.then(doneCallback);
});
}
// Set UI state to updating styles.
function setUpdatingState() {
statusDiv.innerText = 'Updating arrangements...';
controls.setAttribute('disabled', true);
}
// Set UI state to updating instruments.
function setLoadingState() {
statusDiv.innerText = 'Loading samples...';
controls.setAttribute('disabled', true);
chordsContainer.setAttribute('disabled', true);
changeChordsButton.innerText = 'Change chords';
}
// Set UI state to playing.
function setStoppedState() {
statusDiv.innerText = 'Ready to play!';
statusDiv.classList.remove('loading');
controls.removeAttribute('disabled');
chordsContainer.setAttribute('disabled', true);
changeChordsButton.innerText = 'Change chords';
playButton.innerText = 'Play';
chordInputs.forEach(c => c.classList.remove('playing'));
}
// Set UI state to playing.
function setPlayingState() {
statusDiv.innerText = 'Move the slider to interpolate between styles.';
playButton.innerText = 'Stop';
controls.removeAttribute('disabled');
chordsContainer.setAttribute('disabled', true);
changeChordsButton.innerText = 'Change chords';
}
// Set UI state to changing chords.
function setChordChangeState() {
statusDiv.innerText = 'Change chords (triads only) then press Done.';
changeChordsButton.innerText = 'Done';
chordsContainer.removeAttribute('disabled');
chordInputs.forEach(c => c.classList.remove('playing'));
}
// Play the interpolated sequence for the current slider position.
function playProgression(chordIdx) {
const idx = alphaSlider.value;
chordInputs.forEach(c => c.classList.remove('playing'));
chordInputs[chordIdx].classList.add('playing');
const unquantizedSeq = mm.sequences.unquantizeSequence(chordSeqs[idx][chordIdx]);
player.start(humanize(unquantizedSeq))
.then(() => {
const nextChordIdx = (chordIdx + 1) % numChords;
playProgression(nextChordIdx);
});
}
// Update the start style.
function updateSample1() {
playing = false;
setUpdatingState();
player.stop();
setTimeout(() => {
generateSample(z => {
z1 = z;
generateProgressions(setStoppedState);
});
}, 0);
}
// Update the end style.
function updateSample2() {
playing = false;
setUpdatingState();
player.stop();
setTimeout(() => {
generateSample(z => {
z2 = z;
generateProgressions(setStoppedState);
});
}, 0);
}
// Save sequence as MIDI.
function saveSequence() {
const idx = alphaSlider.value;
const midi = mm.sequenceProtoToMidi(progSeqs[idx]);
const file = new Blob([midi], {type: 'audio/midi'});
if (window.navigator.msSaveOrOpenBlob) {
window.navigator.msSaveOrOpenBlob(file, 'prog.mid');
} else { // Others
const a = document.createElement('a');
const url = URL.createObjectURL(file);
a.href = url;
a.download = 'prog.mid';
document.body.appendChild(a);
a.click();
setTimeout(() => {
document.body.removeChild(a);
window.URL.revokeObjectURL(url);
}, 0);
}
}
// Start or stop playing the sequence at the current slider position.
function togglePlaying() {
mm.Player.tone.context.resume();
if (playing) {
playing = false;
setStoppedState();
player.stop();
} else {
playing = true;
setPlayingState();
playProgression(0);
}
}
// Start or finish changing chords.
function toggleChangeChords() {
if (changingChords) {
changingChords = false;
chords = chordInputs.map(c => c.value);
setUpdatingState();
setTimeout(() => generateProgressions(setStoppedState), 0);
} else {
playing = false;
changingChords = true;
setChordChangeState();
player.stop();
}
}
// One of the chords has been edited.
function chordChanged() {
const isGood = (chord) => {
if (!chord) {
return false;
}
try {
mm.chords.ChordSymbols.pitches(chord);
return true;
} catch(e) {
return false;
}
}
var allGood = true;
chordInputs.forEach(c => {
if (isGood(c.value)) {
c.classList.remove('invalid');
} else {
c.classList.add('invalid');
allGood = false;
}
});
changeChordsButton.disabled = !allGood;
}
function initPlayerAndEffects() {
const MAX_PAN = 0.2;
const MIN_DRUM = 35;
const MAX_DRUM = 81;
// Set up effects chain.
const globalCompressor = new mm.Player.tone.MultibandCompressor();
const globalReverb = new mm.Player.tone.Freeverb(0.25);
const globalLimiter = new mm.Player.tone.Limiter();
globalCompressor.connect(globalReverb);
globalReverb.connect(globalLimiter);
globalLimiter.connect(mm.Player.tone.Master);
// Set up per-program effects.
const programMap = new Map();
for (let i = 0; i < 128; i++) {
const programCompressor = new mm.Player.tone.Compressor();
const pan = 2 * MAX_PAN * Math.random() - MAX_PAN;
const programPanner = new mm.Player.tone.Panner(pan);
programMap.set(i, programCompressor);
programCompressor.connect(programPanner);
programPanner.connect(globalCompressor);
}
// Set up per-drum effects.
const drumMap = new Map();
for (let i = MIN_DRUM; i <= MAX_DRUM; i++) {
const drumCompressor = new mm.Player.tone.Compressor();
const pan = 2 * MAX_PAN * Math.random() - MAX_PAN;
const drumPanner = new mm.Player.tone.Panner(pan);
drumMap.set(i, drumCompressor);
drumCompressor.connect(drumPanner);
drumPanner.connect(globalCompressor);
}
// Set up SoundFont player.
const player = new mm.SoundFontPlayer(
'https://storage.googleapis.com/download.magenta.tensorflow.org/soundfonts_js/sgm_plus',
globalCompressor, programMap, drumMap);
return player;
}
@import url(https://fonts.googleapis.com/css?family=Roboto);
* {box-sizing: border-box; }
body {
background: linear-gradient(to right bottom, white 50%, #BCC4EA 50%);
height: 100vh;
margin: 0;
font-family: 'IBM Plex Mono', monospace;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
h1, h2 {
color: #203EC1;
text-align: center;
}
a:link, a:visited {
color: #203EC1;
font-weight: bold;
}
[disabled] {
pointer-events: none;
opacity: 0.3;
}
.content {
background: white;
max-width: 800px;
margin: 40px auto 20px auto;
border: 10px solid #203EC1;
position: relative;
}
.fineprint {
max-width: 800px;
margin: 0px auto;
padding: 40px;
}
.content:after {
content: '';
display: block;
position: absolute;
bottom: -30px;
left: -10px;
width: calc(100% - 60px);
margin: auto;
border-left: 40px solid transparent;
border-right: 40px solid transparent;
border-top: 20px solid #203EC1;
}
#status {
text-align: center;
font-weight: bold;
}
.loading {
animation: pulsing-fade 1.2s ease-in-out infinite;
}
@keyframes pulsing-fade {
50% {
opacity: 0.3;
}
}
.preamble, .bottom {
padding: 20px;
}
.about {
position: relative;
margin-bottom: 30px;
}
.bottom {
background: #F1F3F9;
}
.horizontal {
display: flex;
flex-direction: row;
justify-content: space-between;
align-items: center;
}
.chords {
display: flex;
flex-direction: column;
justify-content: space-between;
align-items: center;
flex-grow: 1;
padding: 0 24px;
}
#chordsContainer[disabled] {
opacity: 1;
}
#chordsContainer[disabled] input {
background: transparent;
border-color: transparent;
}
input.playing {
background: #203EC1 !important;
color: white !important;
}
button {
background: transparent;
border: none;
color: #203EC1;
border: 4px solid #203EC1;
font-size: 14px;
text-transform: uppercase;
letter-spacing: 1px;
padding: 8px 14px;
font-weight: bold;
cursor: pointer;
transition: all 0.2s linear;
}
button:hover {
background: #203EC1;
color: white;
}
button.inverted {
background: #203EC1;
color: white;
border: 4px solid transparent;
font-size: 14px;
text-transform: uppercase;
letter-spacing: 1px;
padding: 8px 14px;
font-weight: bold;
cursor: pointer;
transition: all 0.2s linear;
}
button.inverted:hover {
background: transparent;
border: 4px solid #203EC1;
color: #203EC1;
}
input[type=text] {
background: white;
color: #203EC1;
font-size: 14px;
font-weight: bold;
text-align: center;
letter-spacing: 1px;
padding: 8px;
width: 50px;
border: 1px solid #BDC4E7;
border-radius: 3px;
}
input.invalid {
border: 1px solid red;
color: red;
}
input[type=range] {
margin: 8px 20px;
background: transparent;
flex-grow: 1;
width: 100%;
}
@media screen and (max-width: 500px) {
.preamble, .bottom {
padding: 24px;
}
.horizontal {
flex-direction: column;
padding: 24px 0;
}
.horizontal h2 {
display: inline-block;
}
.bottom {
text-align: center;
}
#chordsContainer {
display: block;
margin: 8px 0;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment