Skip to content

Commit

Permalink
Song update functionality (#304)
Browse files Browse the repository at this point in the history 
* add hopo_threshold support to please jnack

* fix DXT3 image byte parsing

* DXT3 loop cleanup

* hopo threshold fix

* actually reference hopo_threshold this time

* small RB fixes

* song metadata updates

* image updates

* mogg dta updates

* midi updates implemented

* mogg updates implemented

* attempted cleanup of update code

* remove debug statements and fix con updating

* remove debug statement

* fix song scanner not reading midi preparser properly
  • Loading branch information
@rjkiv
rjkiv authored May 12, 2023
1 parent 016bbb6 commit 2886324
Show file tree
Hide file tree
Showing 13 changed files with 409 additions and 109 deletions.
6 changes: 4 additions & 2 deletions Assets/Script/Audio/Bass/BassAudioManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,10 @@ public void LoadMogg(ExtractedConSongEntry exConSong, bool isSpeedUp) {

byte[] moggArray;
if (exConSong is ConSongEntry conSong) {
moggArray = XboxCONInnerFileRetriever.RetrieveFile(conSong.Location,
conSong.MoggFileSize, conSong.MoggFileMemBlockOffsets)[conSong.MoggAddressAudioOffset..];
if(!conSong.UsingUpdateMogg)
moggArray = XboxCONInnerFileRetriever.RetrieveFile(conSong.Location,
conSong.MoggFileSize, conSong.MoggFileMemBlockOffsets)[conSong.MoggAddressAudioOffset..];
else moggArray = File.ReadAllBytes(conSong.MoggPath)[conSong.MoggAddressAudioOffset..];
} else {
moggArray = File.ReadAllBytes(exConSong.MoggPath)[exConSong.MoggAddressAudioOffset..];
}
Expand Down
83 changes: 69 additions & 14 deletions Assets/Script/Serialization/Parser/MidiParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ private struct EventIR {
public MidiFile midi;

public MidiParser(SongEntry songEntry, string[] files) : base(songEntry, files) {
// get base midi
if (songEntry.SongType == SongType.RbCon) {
var conSong = (ConSongEntry) songEntry;
using var stream = new MemoryStream(XboxCONInnerFileRetriever.RetrieveFile(
Expand All @@ -32,27 +33,81 @@ public MidiParser(SongEntry songEntry, string[] files) : base(songEntry, files)
midi = MidiFile.Read(stream, new ReadingSettings() { TextEncoding = System.Text.Encoding.UTF8 });
} else midi = MidiFile.Read(files[0], new ReadingSettings() { TextEncoding = System.Text.Encoding.UTF8 });

// TODO: fix this to account for upgrade CONs/ExCONs
// Merge midi files
for (int i = 1; i < files.Length; i++) {
var upgrade = MidiFile.Read(files[i], new ReadingSettings() { TextEncoding = System.Text.Encoding.UTF8 });
// if this is a RB song, and it contains an update, merge the base and update midi
if(songEntry is ExtractedConSongEntry oof){
if(oof.DiscUpdate){
List<string> BaseTracksToAdd = new List<string>();
List<string> UpdateTracksToAdd = new List<string>();
MidiFile midi_update = MidiFile.Read(oof.UpdateMidiPath, new ReadingSettings() { TextEncoding = System.Text.Encoding.UTF8 });

// get base track names
foreach(var trackChunk in midi.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
BaseTracksToAdd.Add(trackName.Text);
}
}

foreach (var trackChunk in upgrade.GetTrackChunks()) {
foreach (var trackEvent in trackChunk.Events) {
if (trackEvent is not SequenceTrackNameEvent trackName) {
continue;
// get update track names
if(oof.DiscUpdate){
foreach(var trackChunk in midi_update.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
UpdateTracksToAdd.Add(trackName.Text);
// if a track is in both base and update, use the update track
if(BaseTracksToAdd.Find(s => s == trackName.Text) != null) BaseTracksToAdd.Remove(trackName.Text);
}
}
}

// Only merge specific tracks
switch (trackName.Text) {
case "PART REAL_GUITAR":
case "PART REAL_BASS":
midi.Chunks.Add(trackChunk);
break;
UpdateTracksToAdd.RemoveAt(0); // we want to stick with the base midi's tempomap

// create new midi to use and set the tempo map to the base midi's
MidiFile midi_merged = new MidiFile();
midi_merged.ReplaceTempoMap(midi.GetTempoMap());

// first, add approved base tracks to midi_merged
foreach(var trackChunk in midi.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
if(BaseTracksToAdd.Find(s => s == trackName.Text) != null) midi_merged.Chunks.Add(trackChunk);
}
}
// then, the update tracks
foreach(var trackChunk in midi_update.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
if(UpdateTracksToAdd.Find(s => s == trackName.Text) != null) midi_merged.Chunks.Add(trackChunk);
}
}

// finally, assign this new midi as the midi to use in-game
midi = midi_merged;
}

}

// // TODO: fix this to account for upgrade CONs/ExCONs
// // Merge midi files
// for (int i = 1; i < files.Length; i++) {
// var upgrade = MidiFile.Read(files[i], new ReadingSettings() { TextEncoding = System.Text.Encoding.UTF8 });

// foreach (var trackChunk in upgrade.GetTrackChunks()) {
// foreach (var trackEvent in trackChunk.Events) {
// if (trackEvent is not SequenceTrackNameEvent trackName) {
// continue;
// }

// // Only merge specific tracks
// switch (trackName.Text) {
// case "PART REAL_GUITAR":
// case "PART REAL_BASS":
// midi.Chunks.Add(trackChunk);
// break;
// }
// }
// }
// }
}

public override void Parse(YargChart chart) {
Expand Down
102 changes: 59 additions & 43 deletions Assets/Script/Serialization/Xbox/MoggBassInfoGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,54 +11,70 @@

namespace YARG.Serialization {
public static class MoggBASSInfoGenerator {
public static void Generate(ConSongEntry song, DataArray dta){
public static void Generate(ConSongEntry song, DataArray dta, DataArray dta_update_root){
var Tracks = new Dictionary<string, int[]>();
float[] PanData = null, VolumeData = null;
int[] CrowdChannels = null;
int ChannelCount = 0;

for (int i = 1; i < dta.Count; i++) {
var dtaArray = (DataArray) dta[i];
switch (dtaArray[0].ToString()) {
case "tracks":
var trackArray = (DataArray) dtaArray[1];
for (int x = 0; x < trackArray.Count; x++) {
if (trackArray[x] is not DataArray instrArray) continue;
string key = ((DataSymbol) instrArray[0]).Name;
int[] val;
if (instrArray[1] is DataArray trackNums) {
if (trackNums.Count <= 0) continue;
val = new int[trackNums.Count];
for (int y = 0; y < trackNums.Count; y++)
val[y] = ((DataAtom) trackNums[y]).Int;
Tracks.Add(key, val);
} else if (instrArray[1] is DataAtom trackNum) {
val = new int[1];
val[0] = trackNum.Int;
Tracks.Add(key, val);
DataArray dta_update;
List<DataArray> dtas_to_parse = new List<DataArray>();
dtas_to_parse.Add(dta);

// determine whether or not we even NEED to parse the update dta for mogg information
if(dta_update_root != null){
dta_update = dta_update_root.Array("song");
if(dta_update != null){
if(dta_update.Array("tracks") != null || dta_update.Array("pans") != null ||
dta_update.Array("vols") != null || dta_update.Array("crowd_channels") != null)
dtas_to_parse.Add(dta_update);
}
}

foreach(var dta_to_parse in dtas_to_parse){
for (int i = 1; i < dta_to_parse.Count; i++) {
var dtaArray = (DataArray) dta_to_parse[i];
switch (dtaArray[0].ToString()) {
case "tracks":
Tracks.Clear();
var trackArray = (DataArray) dtaArray[1];
for (int x = 0; x < trackArray.Count; x++) {
if (trackArray[x] is not DataArray instrArray) continue;
string key = ((DataSymbol) instrArray[0]).Name;
int[] val;
if (instrArray[1] is DataArray trackNums) {
if (trackNums.Count <= 0) continue;
val = new int[trackNums.Count];
for (int y = 0; y < trackNums.Count; y++)
val[y] = ((DataAtom) trackNums[y]).Int;
Tracks.Add(key, val);
} else if (instrArray[1] is DataAtom trackNum) {
val = new int[1];
val[0] = trackNum.Int;
Tracks.Add(key, val);
}
}
}
break;
case "pans":
var panArray = dtaArray[1] as DataArray;
PanData = new float[panArray.Count];
for (int p = 0; p < panArray.Count; p++) PanData[p] = ((DataAtom) panArray[p]).Float;
ChannelCount = panArray.Count;
break;
case "vols":
var volArray = dtaArray[1] as DataArray;
VolumeData = new float[volArray.Count];
for (int v = 0; v < volArray.Count; v++){
var volAtom = (DataAtom) volArray[v];
if(volAtom.Type == DataType.FLOAT) VolumeData[v] = ((DataAtom) volArray[v]).Float;
else VolumeData[v] = ((DataAtom) volArray[v]).Int;
}
break;
case "crowd_channels":
CrowdChannels = new int[dtaArray.Count - 1];
for (int cc = 1; cc < dtaArray.Count; cc++)
CrowdChannels[cc - 1] = ((DataAtom) dtaArray[cc]).Int;
break;
break;
case "pans":
var panArray = dtaArray[1] as DataArray;
PanData = new float[panArray.Count];
for (int p = 0; p < panArray.Count; p++) PanData[p] = ((DataAtom) panArray[p]).Float;
ChannelCount = panArray.Count;
break;
case "vols":
var volArray = dtaArray[1] as DataArray;
VolumeData = new float[volArray.Count];
for (int v = 0; v < volArray.Count; v++){
var volAtom = (DataAtom) volArray[v];
if(volAtom.Type == DataType.FLOAT) VolumeData[v] = ((DataAtom) volArray[v]).Float;
else VolumeData[v] = ((DataAtom) volArray[v]).Int;
}
break;
case "crowd_channels":
CrowdChannels = new int[dtaArray.Count - 1];
for (int cc = 1; cc < dtaArray.Count; cc++)
CrowdChannels[cc - 1] = ((DataAtom) dtaArray[cc]).Int;
break;
}
}
}

Expand Down
86 changes: 72 additions & 14 deletions Assets/Script/Serialization/Xbox/XboxCONFileBrowser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@

namespace YARG.Serialization {
public static class XboxCONFileBrowser {
public static List<ConSongEntry> BrowseCON(string conName){
public static List<ConSongEntry> BrowseCON(string conName, string update_folder, List<string> update_shortnames){
var songList = new List<ConSongEntry>();
var dtaTree = new DataArray();
var dtaUpdateTree = new DataArray();

// Attempt to read songs.dta
STFS theCON = new STFS(conName);
Expand All @@ -24,43 +25,100 @@ public static List<ConSongEntry> BrowseCON(string conName){
return null;
}

// Attempt to read songs_updates.dta, if it exists
if(update_folder != string.Empty){
try {
using var sr_upd = new StreamReader(Path.Combine(update_folder, "songs_updates.dta"), Encoding.GetEncoding("iso-8859-1"));
dtaUpdateTree = DTX.FromDtaString(sr_upd.ReadToEnd());
} catch (Exception e_upd) {
Debug.LogError($"Failed to parse songs_updates.dta for `{update_folder}`.");
Debug.LogException(e_upd);
return null;
}
}

// Read each song the dta file lists
for (int i = 0; i < dtaTree.Count; i++) {
try {
var currentArray = (DataArray) dtaTree[i];
// Parse songs.dta
// Get song metadata from songs.dta
ConSongEntry currentSong = XboxDTAParser.ParseFromDta(currentArray);


// check if song has applicable updates
bool songCanBeUpdated = (!String.IsNullOrEmpty(update_folder) && (update_shortnames.Find(s => s == currentSong.ShortName) != null));

// if shortname was found in songs_updates.dta, update the metadata
if(songCanBeUpdated)
currentSong = XboxDTAParser.ParseFromDta(dtaUpdateTree.Array(currentSong.ShortName), currentSong);

// since Location is currently set to the name of the folder before mid/mogg/png, set those paths now
// since we're dealing with a CON and not an ExCON, grab each relevant file's sizes and memory block offsets

// capture base midi, and if an update midi was provided, capture that as well
currentSong.NotesFile = Path.Combine("songs", currentSong.Location, $"{currentSong.Location}.mid");
currentSong.MidiFileSize = theCON.GetFileSize(currentSong.NotesFile);
currentSong.MidiFileMemBlockOffsets = theCON.GetMemOffsets(currentSong.NotesFile);
if(songCanBeUpdated && currentSong.DiscUpdate){
string updateMidiPath = Path.Combine(update_folder, currentSong.ShortName, $"{currentSong.ShortName}_update.mid");
if(File.Exists(updateMidiPath)) currentSong.UpdateMidiPath = updateMidiPath;
else {
Debug.LogError($"Couldn't update song {currentSong.ShortName} - update file {currentSong.UpdateMidiPath} not found!");
currentSong.DiscUpdate = false; // to prevent breaking in-game if the user still tries to play the song
}
}

currentSong.MoggPath = Path.Combine("songs", currentSong.Location, $"{currentSong.Location}.mogg");
currentSong.MoggFileSize = theCON.GetFileSize(currentSong.MoggPath);
currentSong.MoggFileMemBlockOffsets = theCON.GetMemOffsets(currentSong.MoggPath);

// capture base mogg path, OR, if update mogg was found, capture that instead
if(songCanBeUpdated){
string updateMoggPath = Path.Combine(update_folder, currentSong.ShortName, $"{currentSong.ShortName}_update.mogg");
if(File.Exists(updateMoggPath)){
currentSong.UsingUpdateMogg = true;
currentSong.MoggPath = updateMoggPath;
}
}
if(!currentSong.UsingUpdateMogg){
currentSong.MoggPath = Path.Combine("songs", currentSong.Location, $"{currentSong.Location}.mogg");
currentSong.MoggFileSize = theCON.GetFileSize(currentSong.MoggPath);
currentSong.MoggFileMemBlockOffsets = theCON.GetMemOffsets(currentSong.MoggPath);
}

// capture base image (if one was provided), OR if update image was found, capture that instead
string imgPath = Path.Combine("songs", currentSong.Location, "gen", $"{currentSong.Location}_keep.png_xbox");
currentSong.ImageFileSize = theCON.GetFileSize(imgPath);
currentSong.ImageFileMemBlockOffsets = theCON.GetMemOffsets(imgPath);

if(currentSong.HasAlbumArt && currentSong.ImageFileSize > 0 && currentSong.ImageFileMemBlockOffsets != null)
currentSong.ImagePath = imgPath;
if(songCanBeUpdated){
string imgUpdatePath = Path.Combine(update_folder, currentSong.ShortName, "gen", $"{currentSong.ShortName}_keep.png_xbox");
if(currentSong.HasAlbumArt && currentSong.AlternatePath){
if(File.Exists(imgUpdatePath)) currentSong.ImagePath = imgUpdatePath;
else currentSong.AlternatePath = false;
}
}

// Set this song's "Location" to the path of the CON file
currentSong.Location = conName;

// Parse the mogg
using var fs = new FileStream(conName, FileMode.Open, FileAccess.Read);
using var br = new BinaryReader(fs);
fs.Seek(currentSong.MoggFileMemBlockOffsets[0], SeekOrigin.Begin);
if(!currentSong.UsingUpdateMogg){
using var fs = new FileStream(conName, FileMode.Open, FileAccess.Read);
using var br = new BinaryReader(fs);
fs.Seek(currentSong.MoggFileMemBlockOffsets[0], SeekOrigin.Begin);

currentSong.MoggHeader = br.ReadInt32();
currentSong.MoggAddressAudioOffset = br.ReadInt32();
currentSong.MoggAudioLength = currentSong.MoggFileSize - currentSong.MoggAddressAudioOffset;
}
else{
using var fs = new FileStream(currentSong.MoggPath, FileMode.Open, FileAccess.Read);
using var br = new BinaryReader(fs);

currentSong.MoggHeader = br.ReadInt32();
currentSong.MoggAddressAudioOffset = br.ReadInt32();
currentSong.MoggAudioLength = currentSong.MoggFileSize - currentSong.MoggAddressAudioOffset;
MoggBASSInfoGenerator.Generate(currentSong, currentArray.Array("song"));
currentSong.MoggHeader = br.ReadInt32();
currentSong.MoggAddressAudioOffset = br.ReadInt32();
currentSong.MoggAudioLength = fs.Length - currentSong.MoggAddressAudioOffset;
}

MoggBASSInfoGenerator.Generate(currentSong, currentArray.Array("song"), dtaUpdateTree.Array(currentSong.ShortName));

// Debug.Log($"{currentSong.ShortName}:\nMidi path: {currentSong.NotesFile}\nMogg path: {currentSong.MoggPath}\nImage path: {currentSong.ImagePath}");

Expand Down
28 changes: 26 additions & 2 deletions Assets/Script/Serialization/Xbox/XboxDTAParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@

namespace YARG.Serialization {
public static class XboxDTAParser {
public static ConSongEntry ParseFromDta(DataArray dta){
var cur = new ConSongEntry();
public static ConSongEntry ParseFromDta(DataArray dta, ConSongEntry existing_song = null){
ConSongEntry cur;
if(existing_song != null) cur = existing_song;
else cur = new ConSongEntry();
cur.ShortName = dta.Name;
// Debug.Log($"this shortname: {dta.Name}");
for (int i = 1; i < dta.Count; i++) {
Expand Down Expand Up @@ -127,6 +129,28 @@ public static ConSongEntry ParseFromDta(DataArray dta){
cur.RealBassTuning = new int[4];
for (int b = 0; b < 4; b++) cur.RealBassTuning[b] = ((DataAtom) bassTunes[b]).Int;
break;
case "alternate_path":
if (dtaArray[1] is DataSymbol symAltPath)
cur.AlternatePath = (symAltPath.Name.ToUpper() == "TRUE");
else if (dtaArray[1] is DataAtom atmAltPath)
cur.AlternatePath = (atmAltPath.Int != 0);
break;
case "extra_authoring":
for(int ea = 1; ea < dtaArray.Count; ea++){
if(dtaArray[ea] is DataSymbol symEA){
if(symEA.Name == "disc_update"){
cur.DiscUpdate = true;
break;
}
}
else if(dtaArray[ea] is DataAtom atmEA){
if(atmEA.String == "disc_update"){
cur.DiscUpdate = true;
break;
}
}
}
break;
}
}

Expand Down
Loading

0 comments on commit 2886324

Please sign in to comment.