Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

songs_upgrades implemented #349

Merged
merged 8 commits into from
May 19, 2023
Merged

songs_upgrades implemented #349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8399f3c
starting song upgrades
rjkiv May 18, 2023
054ff12
Merge branch 'dev' of https://github.com/YARC-Official/YARG into song…
rjkiv May 18, 2023
e9d55aa
ex-CONs can now get ex-upgrades
rjkiv May 18, 2023
c558f6c
songs_upgrades folder now captures upgrades within CONs
rjkiv May 18, 2023
c725a36
standalone songs_upgrades work
rjkiv May 18, 2023
f5dfd7c
tweak rawfile browser to accept root folder
rjkiv May 18, 2023
33f46e2
can now read excon upgrades
rjkiv May 18, 2023
4f8919f
can apply upgrades from CON files
rjkiv May 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 27 additions & 31 deletions Assets/Script/Serialization/Parser/MidiParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ public MidiParser(SongEntry songEntry, string[] files) : base(songEntry, files)
}
else midi = MidiFile.Read(files[0], new ReadingSettings() { TextEncoding = Encoding.UTF8 });

// if this is a RB song, and it contains an update, merge the base and update midi
// if this is a RB song...
if(songEntry is ExtractedConSongEntry oof){
//...and it contains an update, merge the base and update midi
if(oof.DiscUpdate){
List<string> BaseTracksToAdd = new List<string>();
List<string> UpdateTracksToAdd = new List<string>();
Expand All @@ -50,18 +51,18 @@ public MidiParser(SongEntry songEntry, string[] files) : base(songEntry, files)
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
BaseTracksToAdd.Add(trackName.Text);
break;
}
}

// get update track names
if(oof.DiscUpdate){
foreach(var trackChunk in midi_update.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
UpdateTracksToAdd.Add(trackName.Text);
// if a track is in both base and update, use the update track
if(BaseTracksToAdd.Find(s => s == trackName.Text) != null) BaseTracksToAdd.Remove(trackName.Text);
}
foreach(var trackChunk in midi_update.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
UpdateTracksToAdd.Add(trackName.Text);
// if a track is in both base and update, use the update track
if(BaseTracksToAdd.Find(s => s == trackName.Text) != null) BaseTracksToAdd.Remove(trackName.Text);
break;
}
}

Expand All @@ -76,43 +77,38 @@ public MidiParser(SongEntry songEntry, string[] files) : base(songEntry, files)
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
if(BaseTracksToAdd.Find(s => s == trackName.Text) != null) midi_merged.Chunks.Add(trackChunk);
break;
}
}
// then, the update tracks
foreach(var trackChunk in midi_update.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
if(UpdateTracksToAdd.Find(s => s == trackName.Text) != null) midi_merged.Chunks.Add(trackChunk);
break;
}
}

// finally, assign this new midi as the midi to use in-game
midi = midi_merged;
}

// also, if this RB song has a pro upgrade, merge it as well
if(oof.SongUpgrade.UpgradeMidiPath != string.Empty){
using var stream = new MemoryStream(oof.SongUpgrade.GetUpgradeMidi());
MidiFile upgrade = MidiFile.Read(stream, new ReadingSettings() { TextEncoding = Encoding.GetEncoding("iso-8859-1") });

foreach(var trackChunk in upgrade.GetTrackChunks()){
foreach(var trackEvent in trackChunk.Events){
if(trackEvent is not SequenceTrackNameEvent trackName) continue;
if(trackName.Text.Contains("PART REAL_GUITAR") || trackName.Text.Contains("PART REAL_BASS")){
midi.Chunks.Add(trackChunk);
}
}
}

}
}

// // TODO: fix this to account for upgrade CONs/ExCONs
// // Merge midi files
// for (int i = 1; i < files.Length; i++) {
// var upgrade = MidiFile.Read(files[i], new ReadingSettings() { TextEncoding = System.Text.Encoding.UTF8 });

// foreach (var trackChunk in upgrade.GetTrackChunks()) {
// foreach (var trackEvent in trackChunk.Events) {
// if (trackEvent is not SequenceTrackNameEvent trackName) {
// continue;
// }

// // Only merge specific tracks
// switch (trackName.Text) {
// case "PART REAL_GUITAR":
// case "PART REAL_BASS":
// midi.Chunks.Add(trackChunk);
// break;
// }
// }
// }
// }
}

public override void Parse(YargChart chart) {
Expand Down
48 changes: 41 additions & 7 deletions Assets/Script/Serialization/Xbox/XboxCONFileBrowser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,37 @@

namespace YARG.Serialization {
public static class XboxCONFileBrowser {
public static List<ConSongEntry> BrowseCON(string conName, string update_folder, Dictionary<string, List<DataArray>> update_dict){
public static List<ConSongEntry> BrowseCON(string conName,
string update_folder, Dictionary<string, List<DataArray>> update_dict,
Dictionary<SongProUpgrade, DataArray> upgrade_dict){
var songList = new List<ConSongEntry>();
var dtaTree = new DataArray();

// Attempt to read songs.dta
STFS theCON = new STFS(conName);

// Attempt to read upgrades.dta, if it exists
if(theCON.GetFileSize(Path.Combine("songs_upgrades", "upgrades.dta")) > 0){
var dtaUpgradeTree = DTX.FromPlainTextBytes(theCON.GetFile(Path.Combine("songs_upgrades", "upgrades.dta")));

// Read each shortname the dta file lists
for (int i = 0; i < dtaUpgradeTree.Count; i++) {
try {
var currentArray = (DataArray) dtaUpgradeTree[i];
var upgr = new SongProUpgrade();
upgr.ShortName = currentArray.Name;
upgr.UpgradeMidiPath = Path.Combine("songs_upgrades", $"{currentArray.Name}_plus.mid");
upgr.CONFilePath = conName;
upgr.UpgradeMidiFileSize = theCON.GetFileSize(upgr.UpgradeMidiPath);
upgr.UpgradeMidiFileMemBlockOffsets = theCON.GetMemOffsets(upgr.UpgradeMidiPath);
upgrade_dict.Add(upgr, currentArray);
} catch (Exception e) {
Debug.Log($"Failed to get upgrade, skipping...");
Debug.LogException(e);
}
}
}

// Attempt to read songs.dta
try {
dtaTree = DTX.FromPlainTextBytes(theCON.GetFile(Path.Combine("songs", "songs.dta")));
} catch (Exception e) {
Expand All @@ -32,15 +57,24 @@ public static List<ConSongEntry> BrowseCON(string conName, string update_folder,
// Get song metadata from songs.dta
ConSongEntry currentSong = XboxDTAParser.ParseFromDta(currentArray);

// check if song has applicable updates
// check if song has applicable updates and/or upgrades
bool songCanBeUpdated = (update_dict.TryGetValue(currentSong.ShortName, out var val));
bool songHasUpgrade = false;
foreach(var upgr in upgrade_dict){
if(upgr.Key.ShortName == currentSong.ShortName){
songHasUpgrade = true;
currentSong.SongUpgrade = upgr.Key;
break;
}
}

// if shortname was found in songs_updates.dta, update the metadata
if(songCanBeUpdated){
foreach(var dtaUpdate in update_dict[currentSong.ShortName]){
if(songCanBeUpdated)
foreach(var dtaUpdate in update_dict[currentSong.ShortName])
currentSong = XboxDTAParser.ParseFromDta(dtaUpdate, currentSong);
}
}

// if shortname was found in upgrades.dta, apply the upgrade metadata (upgrade midi has already been captured)
if(songHasUpgrade) currentSong = XboxDTAParser.ParseFromDta(upgrade_dict[currentSong.SongUpgrade], currentSong);

// since Location is currently set to the name of the folder before mid/mogg/png, set those paths now
// since we're dealing with a CON and not an ExCON, grab each relevant file's sizes and memory block offsets
Expand Down
53 changes: 44 additions & 9 deletions Assets/Script/Serialization/Xbox/XboxRawfileBrowser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,40 @@

namespace YARG.Serialization {
public static class ExCONBrowser {
public static List<ExtractedConSongEntry> BrowseFolder(string folder, string update_folder, Dictionary<string, List<DataArray>> update_dict){
public static List<ExtractedConSongEntry> BrowseFolder(string root_folder,
string update_folder, Dictionary<string, List<DataArray>> update_dict,
Dictionary<SongProUpgrade, DataArray> upgrade_dict){
var songList = new List<ExtractedConSongEntry>();
var dtaTree = new DataArray();
string songs_folder = Path.Combine(root_folder, "songs");
string songs_upgrades_folder = Path.Combine(root_folder, "songs_upgrades"); // TODO: implement this

// capture any extra upgrades local to this excon, if they exist
if(File.Exists(Path.Combine(songs_upgrades_folder, "upgrades.dta"))){
using var sr_upgr = new StreamReader(Path.Combine(songs_upgrades_folder, "upgrades.dta"), Encoding.GetEncoding("iso-8859-1"));
var dtaUpgradeTree = DTX.FromDtaString(sr_upgr.ReadToEnd());

// Read each shortname the dta file lists
for (int i = 0; i < dtaUpgradeTree.Count; i++) {
try {
var currentArray = (DataArray) dtaUpgradeTree[i];
var upgr = new SongProUpgrade();
upgr.ShortName = currentArray.Name;
upgr.UpgradeMidiPath = Path.Combine(songs_upgrades_folder, $"{currentArray.Name}_plus.mid");
upgrade_dict.Add(upgr, currentArray);
} catch (Exception e) {
Debug.Log($"Failed to get upgrade, skipping...");
Debug.LogException(e);
}
}
}

// Attempt to read songs.dta
try {
using var sr = new StreamReader(Path.Combine(folder, "songs.dta"), Encoding.GetEncoding("iso-8859-1"));
using var sr = new StreamReader(Path.Combine(songs_folder, "songs.dta"), Encoding.GetEncoding("iso-8859-1"));
dtaTree = DTX.FromDtaString(sr.ReadToEnd());
} catch (Exception e) {
Debug.LogError($"Failed to parse songs.dta for `{folder}`.");
Debug.LogError($"Failed to parse songs.dta for `{songs_folder}`.");
Debug.LogException(e);
return null;
}
Expand All @@ -32,18 +56,29 @@ public static List<ExtractedConSongEntry> BrowseFolder(string folder, string upd
// Parse songs.dta for song metadata
var currentSong = XboxDTAParser.ParseFromDta(currentArray);

// check if song has applicable updates
// check if song has applicable updates and/or upgrades
bool songCanBeUpdated = (update_dict.TryGetValue(currentSong.ShortName, out var val));
bool songHasUpgrade = false;
foreach(var upgr in upgrade_dict){
if(upgr.Key.ShortName == currentSong.ShortName){
songHasUpgrade = true;
currentSong.SongUpgrade = upgr.Key;
break;
}
}

// if shortname was found in songs_updates.dta, update the metadata
if(songCanBeUpdated)
foreach(var dtaUpdate in update_dict[currentSong.ShortName])
currentSong = XboxDTAParser.ParseFromDta(dtaUpdate, currentSong);


// if shortname was found in upgrades.dta, apply the upgrade metadata (upgrade midi has already been captured)
if(songHasUpgrade) currentSong = XboxDTAParser.ParseFromDta(upgrade_dict[currentSong.SongUpgrade], currentSong);

// since Location is currently set to the name of the folder before mid/mogg/png, set those paths now:

// capture base midi, and if an update midi was provided, capture that as well
currentSong.NotesFile = Path.Combine(folder, currentSong.Location, $"{currentSong.Location}.mid");
currentSong.NotesFile = Path.Combine(songs_folder, currentSong.Location, $"{currentSong.Location}.mid");
if(songCanBeUpdated && currentSong.DiscUpdate){
string updateMidiPath = Path.Combine(update_folder, currentSong.ShortName, $"{currentSong.ShortName}_update.mid");
if(File.Exists(updateMidiPath)) currentSong.UpdateMidiPath = updateMidiPath;
Expand All @@ -54,7 +89,7 @@ public static List<ExtractedConSongEntry> BrowseFolder(string folder, string upd
}

// capture base mogg path, OR, if update mogg was found, capture that instead
currentSong.MoggPath = Path.Combine(folder, currentSong.Location, $"{currentSong.Location}.mogg");
currentSong.MoggPath = Path.Combine(songs_folder, currentSong.Location, $"{currentSong.Location}.mogg");
if(songCanBeUpdated){
string updateMoggPath = Path.Combine(update_folder, currentSong.ShortName, $"{currentSong.ShortName}_update.mogg");
if(File.Exists(updateMoggPath)){
Expand All @@ -64,7 +99,7 @@ public static List<ExtractedConSongEntry> BrowseFolder(string folder, string upd
}

// capture base image (if one was provided), OR if update image was found, capture that instead
string imgPath = Path.Combine(folder, currentSong.Location, "gen", $"{currentSong.Location}_keep.png_xbox");
string imgPath = Path.Combine(songs_folder, currentSong.Location, "gen", $"{currentSong.Location}_keep.png_xbox");
if(currentSong.HasAlbumArt && File.Exists(imgPath))
currentSong.ImagePath = imgPath;
if(songCanBeUpdated){
Expand All @@ -76,7 +111,7 @@ public static List<ExtractedConSongEntry> BrowseFolder(string folder, string upd
}

// Get song folder path for mid, mogg, png_xbox
currentSong.Location = Path.Combine(folder, currentSong.Location);
currentSong.Location = Path.Combine(songs_folder, currentSong.Location);

// Parse the mogg
using var fs = new FileStream(currentSong.MoggPath, FileMode.Open, FileAccess.Read);
Expand Down
89 changes: 89 additions & 0 deletions Assets/Script/Serialization/Xbox/XboxSongUpgradeBrowser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using DtxCS;
using DtxCS.DataTypes;
using UnityEngine;
using XboxSTFS;
using YARG.Data;
using YARG.Song;

namespace YARG.Serialization {
public static class XboxSongUpgradeBrowser {
public static Dictionary<SongProUpgrade, DataArray> FetchSongUpgrades(string upgrade_folder){
var dtaTree = new DataArray();
var UpgradeSongDict = new Dictionary<SongProUpgrade, DataArray>();

// TODO: tweak this function so you parse raw upgrades, and THEN upgrades contained within CONs
// FIRST, parse raw upgrades - start by attempting to read upgrades.dta
if(File.Exists(Path.Combine(upgrade_folder, "upgrades.dta"))){
using var sr = new StreamReader(Path.Combine(upgrade_folder, "upgrades.dta"), Encoding.GetEncoding("iso-8859-1"));
dtaTree = DTX.FromDtaString(sr.ReadToEnd());

// Read each shortname the dta file lists
for (int i = 0; i < dtaTree.Count; i++) {
try {
var currentArray = (DataArray) dtaTree[i];
var upgr = new SongProUpgrade();
upgr.ShortName = currentArray.Name;
upgr.UpgradeMidiPath = Path.Combine(upgrade_folder, $"{currentArray.Name}_plus.mid");
UpgradeSongDict.Add(upgr, currentArray);
} catch (Exception e) {
Debug.Log($"Failed to get upgrade, skipping...");
Debug.LogException(e);
}
}
}

// THEN, find any loose CONs in this directory and parse those for upgrades as well
foreach (var file in Directory.EnumerateFiles(upgrade_folder)) {
if(Path.GetExtension(file) != ".mid" && Path.GetExtension(file) != ".dta"){
// for each file found, read first 4 bytes and check for "CON " or "LIVE"
using var fs = new FileStream(file, FileMode.Open, FileAccess.Read);
using var br = new BinaryReader(fs);
string fHeader = Encoding.UTF8.GetString(br.ReadBytes(4));
if (fHeader == "CON " || fHeader == "LIVE") {
STFS thisUpgradeCON = new STFS(file);

// attempt to read the CON's upgrades.dta
try {
dtaTree = DTX.FromPlainTextBytes(thisUpgradeCON.GetFile(Path.Combine("songs_upgrades", "upgrades.dta")));
} catch (Exception e) {
Debug.LogError($"Failed to parse upgrades.dta for `{file}`.");
Debug.LogException(e);
continue;
}

// Read each shortname the dta file lists
for(int i = 0; i < dtaTree.Count; i++){
try {
var currentArray = (DataArray) dtaTree[i];
var upgr = new SongProUpgrade();
upgr.ShortName = currentArray.Name;
upgr.UpgradeMidiPath = Path.Combine("songs_upgrades", $"{currentArray.Name}_plus.mid");
upgr.CONFilePath = file;
upgr.UpgradeMidiFileSize = thisUpgradeCON.GetFileSize(upgr.UpgradeMidiPath);
upgr.UpgradeMidiFileMemBlockOffsets = thisUpgradeCON.GetMemOffsets(upgr.UpgradeMidiPath);
UpgradeSongDict.Add(upgr, currentArray);
} catch (Exception e) {
Debug.Log($"Failed to get upgrade, skipping...");
Debug.LogException(e);
}
}

}
}
}

// Debug.Log($"Song upgrades:");
// foreach(var item in UpgradeSongDict){
// Debug.Log($"{item.Key.ShortName} has a pro upgrade");
// }

return UpgradeSongDict;

}
}
}
11 changes: 11 additions & 0 deletions Assets/Script/Serialization/Xbox/XboxSongUpgradeBrowser.cs.meta
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading