Skip to content

Commit

Permalink
songs_upgrades implemented (#349)
Browse files Browse the repository at this point in the history
* starting song upgrades

* ex-CONs can now get ex-upgrades

* songs_upgrades folder now captures upgrades within CONs

* standalone songs_upgrades work

* tweak rawfile browser to accept root folder

* can now read excon upgrades

* can apply upgrades from CON files
  • Loading branch information
rjkiv authored May 19, 2023
1 parent e61df69 commit 5b308fd
Show file tree
Hide file tree
Showing 10 changed files with 345 additions and 55 deletions.
58 changes: 27 additions & 31 deletions Assets/Script/Serialization/Parser/MidiParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ public MidiParser(SongEntry songEntry, string[] files) : base(songEntry, files)
}
else midi = MidiFile.Read(files[0], new ReadingSettings() { TextEncoding = Encoding.UTF8 });

// if this is a RB song, and it contains an update, merge the base and update midi
// if this is a RB song...
if(songEntry is ExtractedConSongEntry oof){
//...and it contains an update, merge the base and update midi
if(oof.DiscUpdate){
List<string> BaseTracksToAdd = new List<string>();
List<string> UpdateTracksToAdd = new List<string>();
Expand All @@ -50,18 +51,18 @@ public MidiParser(SongEntry songEntry, string[] files) : base(songEntry, files)
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
BaseTracksToAdd.Add(trackName.Text);
break;
}
}

// get update track names
if(oof.DiscUpdate){
foreach(var trackChunk in midi_update.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
UpdateTracksToAdd.Add(trackName.Text);
// if a track is in both base and update, use the update track
if(BaseTracksToAdd.Find(s => s == trackName.Text) != null) BaseTracksToAdd.Remove(trackName.Text);
}
foreach(var trackChunk in midi_update.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
UpdateTracksToAdd.Add(trackName.Text);
// if a track is in both base and update, use the update track
if(BaseTracksToAdd.Find(s => s == trackName.Text) != null) BaseTracksToAdd.Remove(trackName.Text);
break;
}
}

Expand All @@ -76,43 +77,38 @@ public MidiParser(SongEntry songEntry, string[] files) : base(songEntry, files)
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
if(BaseTracksToAdd.Find(s => s == trackName.Text) != null) midi_merged.Chunks.Add(trackChunk);
break;
}
}
// then, the update tracks
foreach(var trackChunk in midi_update.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
if(UpdateTracksToAdd.Find(s => s == trackName.Text) != null) midi_merged.Chunks.Add(trackChunk);
break;
}
}

// finally, assign this new midi as the midi to use in-game
midi = midi_merged;
}

// also, if this RB song has a pro upgrade, merge it as well
if(oof.SongUpgrade.UpgradeMidiPath != string.Empty){
using var stream = new MemoryStream(oof.SongUpgrade.GetUpgradeMidi());
MidiFile upgrade = MidiFile.Read(stream, new ReadingSettings() { TextEncoding = Encoding.GetEncoding("iso-8859-1") });

foreach(var trackChunk in upgrade.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
if(trackName.Text.Contains("PART REAL_GUITAR") || trackName.Text.Contains("PART REAL_BASS")){
midi.Chunks.Add(trackChunk);
}
}
}

}
}

// // TODO: fix this to account for upgrade CONs/ExCONs
// // Merge midi files
// for (int i = 1; i < files.Length; i++) {
// var upgrade = MidiFile.Read(files[i], new ReadingSettings() { TextEncoding = System.Text.Encoding.UTF8 });

// foreach (var trackChunk in upgrade.GetTrackChunks()) {
// foreach (var trackEvent in trackChunk.Events) {
// if (trackEvent is not SequenceTrackNameEvent trackName) {
// continue;
// }

// // Only merge specific tracks
// switch (trackName.Text) {
// case "PART REAL_GUITAR":
// case "PART REAL_BASS":
// midi.Chunks.Add(trackChunk);
// break;
// }
// }
// }
// }
}

public override void Parse(YargChart chart) {
Expand Down
48 changes: 41 additions & 7 deletions Assets/Script/Serialization/Xbox/XboxCONFileBrowser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,37 @@

namespace YARG.Serialization {
public static class XboxCONFileBrowser {
public static List<ConSongEntry> BrowseCON(string conName, string update_folder, Dictionary<string, List<DataArray>> update_dict){
public static List<ConSongEntry> BrowseCON(string conName,
string update_folder, Dictionary<string, List<DataArray>> update_dict,
Dictionary<SongProUpgrade, DataArray> upgrade_dict){
var songList = new List<ConSongEntry>();
var dtaTree = new DataArray();

// Attempt to read songs.dta
STFS theCON = new STFS(conName);

// Attempt to read upgrades.dta, if it exists
if(theCON.GetFileSize(Path.Combine("songs_upgrades", "upgrades.dta")) > 0){
var dtaUpgradeTree = DTX.FromPlainTextBytes(theCON.GetFile(Path.Combine("songs_upgrades", "upgrades.dta")));

// Read each shortname the dta file lists
for (int i = 0; i < dtaUpgradeTree.Count; i++) {
try {
var currentArray = (DataArray) dtaUpgradeTree[i];
var upgr = new SongProUpgrade();
upgr.ShortName = currentArray.Name;
upgr.UpgradeMidiPath = Path.Combine("songs_upgrades", $"{currentArray.Name}_plus.mid");
upgr.CONFilePath = conName;
upgr.UpgradeMidiFileSize = theCON.GetFileSize(upgr.UpgradeMidiPath);
upgr.UpgradeMidiFileMemBlockOffsets = theCON.GetMemOffsets(upgr.UpgradeMidiPath);
upgrade_dict.Add(upgr, currentArray);
} catch (Exception e) {
Debug.Log($"Failed to get upgrade, skipping...");
Debug.LogException(e);
}
}
}

// Attempt to read songs.dta
try {
dtaTree = DTX.FromPlainTextBytes(theCON.GetFile(Path.Combine("songs", "songs.dta")));
} catch (Exception e) {
Expand All @@ -32,15 +57,24 @@ public static List<ConSongEntry> BrowseCON(string conName, string update_folder,
// Get song metadata from songs.dta
ConSongEntry currentSong = XboxDTAParser.ParseFromDta(currentArray);

// check if song has applicable updates
// check if song has applicable updates and/or upgrades
bool songCanBeUpdated = (update_dict.TryGetValue(currentSong.ShortName, out var val));
bool songHasUpgrade = false;
foreach(var upgr in upgrade_dict){
if(upgr.Key.ShortName == currentSong.ShortName){
songHasUpgrade = true;
currentSong.SongUpgrade = upgr.Key;
break;
}
}

// if shortname was found in songs_updates.dta, update the metadata
if(songCanBeUpdated){
foreach(var dtaUpdate in update_dict[currentSong.ShortName]){
if(songCanBeUpdated)
foreach(var dtaUpdate in update_dict[currentSong.ShortName])
currentSong = XboxDTAParser.ParseFromDta(dtaUpdate, currentSong);
}
}

// if shortname was found in upgrades.dta, apply the upgrade metadata (upgrade midi has already been captured)
if(songHasUpgrade) currentSong = XboxDTAParser.ParseFromDta(upgrade_dict[currentSong.SongUpgrade], currentSong);

// since Location is currently set to the name of the folder before mid/mogg/png, set those paths now
// since we're dealing with a CON and not an ExCON, grab each relevant file's sizes and memory block offsets
Expand Down
53 changes: 44 additions & 9 deletions Assets/Script/Serialization/Xbox/XboxRawfileBrowser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,40 @@

namespace YARG.Serialization {
public static class ExCONBrowser {
public static List<ExtractedConSongEntry> BrowseFolder(string folder, string update_folder, Dictionary<string, List<DataArray>> update_dict){
public static List<ExtractedConSongEntry> BrowseFolder(string root_folder,
string update_folder, Dictionary<string, List<DataArray>> update_dict,
Dictionary<SongProUpgrade, DataArray> upgrade_dict){
var songList = new List<ExtractedConSongEntry>();
var dtaTree = new DataArray();
string songs_folder = Path.Combine(root_folder, "songs");
string songs_upgrades_folder = Path.Combine(root_folder, "songs_upgrades"); // TODO: implement this

// capture any extra upgrades local to this excon, if they exist
if(File.Exists(Path.Combine(songs_upgrades_folder, "upgrades.dta"))){
using var sr_upgr = new StreamReader(Path.Combine(songs_upgrades_folder, "upgrades.dta"), Encoding.GetEncoding("iso-8859-1"));
var dtaUpgradeTree = DTX.FromDtaString(sr_upgr.ReadToEnd());

// Read each shortname the dta file lists
for (int i = 0; i < dtaUpgradeTree.Count; i++) {
try {
var currentArray = (DataArray) dtaUpgradeTree[i];
var upgr = new SongProUpgrade();
upgr.ShortName = currentArray.Name;
upgr.UpgradeMidiPath = Path.Combine(songs_upgrades_folder, $"{currentArray.Name}_plus.mid");
upgrade_dict.Add(upgr, currentArray);
} catch (Exception e) {
Debug.Log($"Failed to get upgrade, skipping...");
Debug.LogException(e);
}
}
}

// Attempt to read songs.dta
try {
using var sr = new StreamReader(Path.Combine(folder, "songs.dta"), Encoding.GetEncoding("iso-8859-1"));
using var sr = new StreamReader(Path.Combine(songs_folder, "songs.dta"), Encoding.GetEncoding("iso-8859-1"));
dtaTree = DTX.FromDtaString(sr.ReadToEnd());
} catch (Exception e) {
Debug.LogError($"Failed to parse songs.dta for `{folder}`.");
Debug.LogError($"Failed to parse songs.dta for `{songs_folder}`.");
Debug.LogException(e);
return null;
}
Expand All @@ -32,18 +56,29 @@ public static List<ExtractedConSongEntry> BrowseFolder(string folder, string upd
// Parse songs.dta for song metadata
var currentSong = XboxDTAParser.ParseFromDta(currentArray);

// check if song has applicable updates
// check if song has applicable updates and/or upgrades
bool songCanBeUpdated = (update_dict.TryGetValue(currentSong.ShortName, out var val));
bool songHasUpgrade = false;
foreach(var upgr in upgrade_dict){
if(upgr.Key.ShortName == currentSong.ShortName){
songHasUpgrade = true;
currentSong.SongUpgrade = upgr.Key;
break;
}
}

// if shortname was found in songs_updates.dta, update the metadata
if(songCanBeUpdated)
foreach(var dtaUpdate in update_dict[currentSong.ShortName])
currentSong = XboxDTAParser.ParseFromDta(dtaUpdate, currentSong);


// if shortname was found in upgrades.dta, apply the upgrade metadata (upgrade midi has already been captured)
if(songHasUpgrade) currentSong = XboxDTAParser.ParseFromDta(upgrade_dict[currentSong.SongUpgrade], currentSong);

// since Location is currently set to the name of the folder before mid/mogg/png, set those paths now:

// capture base midi, and if an update midi was provided, capture that as well
currentSong.NotesFile = Path.Combine(folder, currentSong.Location, $"{currentSong.Location}.mid");
currentSong.NotesFile = Path.Combine(songs_folder, currentSong.Location, $"{currentSong.Location}.mid");
if(songCanBeUpdated && currentSong.DiscUpdate){
string updateMidiPath = Path.Combine(update_folder, currentSong.ShortName, $"{currentSong.ShortName}_update.mid");
if(File.Exists(updateMidiPath)) currentSong.UpdateMidiPath = updateMidiPath;
Expand All @@ -54,7 +89,7 @@ public static List<ExtractedConSongEntry> BrowseFolder(string folder, string upd
}

// capture base mogg path, OR, if update mogg was found, capture that instead
currentSong.MoggPath = Path.Combine(folder, currentSong.Location, $"{currentSong.Location}.mogg");
currentSong.MoggPath = Path.Combine(songs_folder, currentSong.Location, $"{currentSong.Location}.mogg");
if(songCanBeUpdated){
string updateMoggPath = Path.Combine(update_folder, currentSong.ShortName, $"{currentSong.ShortName}_update.mogg");
if(File.Exists(updateMoggPath)){
Expand All @@ -64,7 +99,7 @@ public static List<ExtractedConSongEntry> BrowseFolder(string folder, string upd
}

// capture base image (if one was provided), OR if update image was found, capture that instead
string imgPath = Path.Combine(folder, currentSong.Location, "gen", $"{currentSong.Location}_keep.png_xbox");
string imgPath = Path.Combine(songs_folder, currentSong.Location, "gen", $"{currentSong.Location}_keep.png_xbox");
if(currentSong.HasAlbumArt && File.Exists(imgPath))
currentSong.ImagePath = imgPath;
if(songCanBeUpdated){
Expand All @@ -76,7 +111,7 @@ public static List<ExtractedConSongEntry> BrowseFolder(string folder, string upd
}

// Get song folder path for mid, mogg, png_xbox
currentSong.Location = Path.Combine(folder, currentSong.Location);
currentSong.Location = Path.Combine(songs_folder, currentSong.Location);

// Parse the mogg
using var fs = new FileStream(currentSong.MoggPath, FileMode.Open, FileAccess.Read);
Expand Down
89 changes: 89 additions & 0 deletions Assets/Script/Serialization/Xbox/XboxSongUpgradeBrowser.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using DtxCS;
using DtxCS.DataTypes;
using UnityEngine;
using XboxSTFS;
using YARG.Data;
using YARG.Song;

namespace YARG.Serialization {
public static class XboxSongUpgradeBrowser {
public static Dictionary<SongProUpgrade, DataArray> FetchSongUpgrades(string upgrade_folder){
var dtaTree = new DataArray();
var UpgradeSongDict = new Dictionary<SongProUpgrade, DataArray>();

// TODO: tweak this function so you parse raw upgrades, and THEN upgrades contained within CONs
// FIRST, parse raw upgrades - start by attempting to read upgrades.dta
if(File.Exists(Path.Combine(upgrade_folder, "upgrades.dta"))){
using var sr = new StreamReader(Path.Combine(upgrade_folder, "upgrades.dta"), Encoding.GetEncoding("iso-8859-1"));
dtaTree = DTX.FromDtaString(sr.ReadToEnd());

// Read each shortname the dta file lists
for (int i = 0; i < dtaTree.Count; i++) {
try {
var currentArray = (DataArray) dtaTree[i];
var upgr = new SongProUpgrade();
upgr.ShortName = currentArray.Name;
upgr.UpgradeMidiPath = Path.Combine(upgrade_folder, $"{currentArray.Name}_plus.mid");
UpgradeSongDict.Add(upgr, currentArray);
} catch (Exception e) {
Debug.Log($"Failed to get upgrade, skipping...");
Debug.LogException(e);
}
}
}

// THEN, find any loose CONs in this directory and parse those for upgrades as well
foreach (var file in Directory.EnumerateFiles(upgrade_folder)) {
if(Path.GetExtension(file) != ".mid" && Path.GetExtension(file) != ".dta"){
// for each file found, read first 4 bytes and check for "CON " or "LIVE"
using var fs = new FileStream(file, FileMode.Open, FileAccess.Read);
using var br = new BinaryReader(fs);
string fHeader = Encoding.UTF8.GetString(br.ReadBytes(4));
if (fHeader == "CON " || fHeader == "LIVE") {
STFS thisUpgradeCON = new STFS(file);

// attempt to read the CON's upgrades.dta
try {
dtaTree = DTX.FromPlainTextBytes(thisUpgradeCON.GetFile(Path.Combine("songs_upgrades", "upgrades.dta")));
} catch (Exception e) {
Debug.LogError($"Failed to parse upgrades.dta for `{file}`.");
Debug.LogException(e);
continue;
}

// Read each shortname the dta file lists
for(int i = 0; i < dtaTree.Count; i++){
try {
var currentArray = (DataArray) dtaTree[i];
var upgr = new SongProUpgrade();
upgr.ShortName = currentArray.Name;
upgr.UpgradeMidiPath = Path.Combine("songs_upgrades", $"{currentArray.Name}_plus.mid");
upgr.CONFilePath = file;
upgr.UpgradeMidiFileSize = thisUpgradeCON.GetFileSize(upgr.UpgradeMidiPath);
upgr.UpgradeMidiFileMemBlockOffsets = thisUpgradeCON.GetMemOffsets(upgr.UpgradeMidiPath);
UpgradeSongDict.Add(upgr, currentArray);
} catch (Exception e) {
Debug.Log($"Failed to get upgrade, skipping...");
Debug.LogException(e);
}
}

}
}
}

// Debug.Log($"Song upgrades:");
// foreach(var item in UpgradeSongDict){
// Debug.Log($"{item.Key.ShortName} has a pro upgrade");
// }

return UpgradeSongDict;

}
}
}
11 changes: 11 additions & 0 deletions Assets/Script/Serialization/Xbox/XboxSongUpgradeBrowser.cs.meta

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 5b308fd

Please sign in to comment.