Skip to content

Commit

Permalink
Add support for publisher in multiple languages
Browse files Browse the repository at this point in the history
  • Loading branch information
@jerikso
jerikso committed Nov 2, 2023
1 parent 386c300 commit 83c077d
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 82 deletions.
139 changes: 68 additions & 71 deletions Px.Dcat/DataCollector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -511,75 +511,68 @@ private string getCategory(List<PxMenuItem> path)
return "http://publications.europa.eu/Resource/authority/data-theme/" + _themeMapping[category];
}

/// <summary>
/// Get producer of table
/// </summary>
/// <param name="meta">Metadata of table</param>
/// <returns>Organization with the producer info</returns>
private Organization getProducer(PXMeta meta, List<string> langs)
private void addOrganization(HashSet<(string, string)> names)
{
HashSet<(string, string)> names = new HashSet<(string, string)>();
List<Organization> matchingOrgs = new List<Organization>();
HashSet<(string, string)> newNames = new HashSet<(string, string)>(names);

foreach (string lang in langs)
foreach ((string lang, string name) in names)
{
meta.SetLanguage(lang);
string name = meta.Source;
if (_organizations.ContainsKey(name))
{
matchingOrgs.Add(_organizations[name]);
}
names.Add((lang, name));
}

Organization newOrg = new Organization();
newOrg.Resource = Path.Combine(_settings.BaseUri, "organization", nextString()).Replace("\\", "/");
if (matchingOrgs.Count > 0)
{
foreach (Organization org in matchingOrgs)
foreach (Organization o in matchingOrgs)
{
names.UnionWith(org.Names);
}

newOrg.Names = names;
newOrg.Resource = Path.Combine(_settings.BaseUri,"organization",nextString()).Replace("\\", "/");

foreach (string name in names.Select(x => x.Item2).Distinct())
{
_organizations[name] = newOrg;
newNames.UnionWith(o.Names);
}

}
else
newOrg.Names = newNames;

foreach (string name in newNames.Select(x => x.Item2).Distinct())
{
newOrg.Names = names;
newOrg.Resource = Path.Combine(_settings.BaseUri, "organization", nextString()).Replace("\\", "/");
_organizations[name] = newOrg;
}
}

// Add a reference to the organization for each language
foreach (string name in names.Select(x => x.Item2).Distinct())
{
_organizations.Add(name, newOrg);
}
/// <summary>
/// Get producer of table
/// </summary>
/// <param name="meta">Metadata of table</param>
/// <returns>Organization with the producer info</returns>
private void setProducer(PXMeta meta, List<string> langs)
{
HashSet<(string, string)> names = new HashSet<(string, string)>();

foreach (string lang in langs)
{
meta.SetLanguage(lang);
string name = meta.Source;
names.Add((lang, name));
}

return newOrg;
addOrganization(names);
}

/// <summary>
/// Get publisher from _settings
/// </summary>
/// <returns>Publisher</returns>
private Organization getPublisher()
private void setPublisher()
{
string name = _settings.PublisherName;
Organization org;
if (!_organizations.TryGetValue(name, out org))
HashSet<(string, string)> names = new HashSet<(string, string)>();
foreach (KeyValuePair<string, string> pair in _settings.PublisherNames)
{
HashSet<(string, string)> names = new HashSet<(string, string)>();
names.Add((null, name));
org = new Organization { Names = names, Resource = Path.Combine(_settings.BaseUri, "organization", nextString()).Replace("\\", "/") };
_organizations.Add(name, org);
names.Add((pair.Key, pair.Value));
}
return org;
addOrganization(names);
}

/// <summary>
Expand Down Expand Up @@ -764,7 +757,7 @@ private Dataset getDataset(string selection, PXMeta meta, List<PxMenuItem> path)

dataset.Sources = getSources(meta, langs);

getProducer(meta, langs); // Wait until all organizations are created before assigning producer
setProducer(meta, langs); // Wait until all organizations are created before assigning producer

return dataset;
}
Expand Down Expand Up @@ -855,12 +848,6 @@ private List<Dataset> getDatasets()
Item baseItem = _fetcher.GetBaseItem("", "", _settings.MainLanguage, _settings.DatabaseId);

addRecursive(baseItem, path, datasets);

_publisher = getPublisher();
foreach (Dataset d in datasets)
{
d.Publisher = _publisher;
}
return datasets;
}

Expand All @@ -883,9 +870,16 @@ private Catalog getCatalog()
c.License = _settings.License;
c.Datasets = getDatasets();
c.Languages = convertLanguages(_settings.Languages);
c.Publisher = _publisher;
setPublisher();
setOrganizationResources();
setProducers(c.Datasets);

Organization publisher = _organizations[_settings.PublisherNames.First().Value];
c.Publisher = publisher;
foreach (Dataset d in c.Datasets)
{
d.Publisher = publisher;
}
return c;
}

Expand All @@ -900,44 +894,47 @@ private void setProducers(List<Dataset> datasets)

private void setOrganizationResources()
{
foreach (string source in _organizations.Keys)
Dictionary<string, List<string>> reverseResourceMapping = new Dictionary<string, List<string>>();

foreach (string key in _organizationMapping.Keys)
{
if (_organizationMapping.ContainsKey(source))
string res = _organizationMapping[key];
if (!reverseResourceMapping.ContainsKey(res))
{
_organizations[source].Resource = _organizationMapping[source];
reverseResourceMapping[res] = new List<string>();
}
reverseResourceMapping[res].Add(key);
}

Dictionary<string, Organization> newOrgs = new Dictionary<string, Organization>();

// Merge organizations with same resource
foreach (string source in _organizations.Keys)
// Merge organizations mapped to same resource
foreach (string res in reverseResourceMapping.Keys)
{
Organization org1 = _organizations[source];
string res1 = org1.Resource;
foreach (string source2 in _organizations.Keys)
List<string> sources = reverseResourceMapping[res];
if (sources.Count > 1)
{
if (source == source2) continue;

Organization org2 = _organizations[source2];
string res2 = org2.Resource;

if (res1 == res2)
HashSet<(string, string)> names = new HashSet<(string, string)>();
foreach (string name in sources)
{
names.UnionWith(_organizations[name].Names);
}
Organization newOrg = new Organization();
newOrg.Names = names;
newOrg.Resource = res;
foreach ((string lang, string name) in names)
{
Organization newOrg = new Organization();
newOrg.Resource = res1;
newOrg.Names = new HashSet<(string, string)>(org1.Names.Union(org2.Names));
newOrgs[source] = newOrg;
newOrgs[source2] = newOrg;
_organizations[name] = newOrg;
}

}
}

foreach (string source in newOrgs.Keys)
foreach (string mappedSource in _organizationMapping.Keys)
{
_organizations[source] = newOrgs[source];
if (_organizations.ContainsKey(mappedSource))
{
_organizations[mappedSource].Resource = _organizationMapping[mappedSource];
}
}

}

/// <summary>
Expand Down
2 changes: 1 addition & 1 deletion Px.Dcat/DcatSettings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ public struct DcatSettings
public List<string> Languages; // Read from settings
public List<KeyValuePair<string, string>> CatalogTitles;
public List<KeyValuePair<string, string>> CatalogDescriptions;
public List<KeyValuePair<string, string>> PublisherNames;

public string PublisherName;
public string DatabaseId;
public DatabaseType DatabaseType;
public string LandingPageUrl;
Expand Down
21 changes: 11 additions & 10 deletions TestApp/Main.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,22 @@ public static void Main(string[] args)
DcatSettings settings = new DcatSettings
{
BaseUri = "https://www.baseURI.se/",
BaseApiUrl = "http://api.scb.se/OV0104/v1/doris/",
Languages = new List<string> { "sv", "en" },
//BaseApiUrl = "http://api.scb.se/OV0104/v1/doris/",
BaseApiUrl = "http://localhost:56338/api/v1/",
Languages = new List<string> { "en" },
CatalogTitles = new List<KeyValuePair<string, string>> { new KeyValuePair<string, string>("sv", "SCB Tabeller"), new KeyValuePair<string, string>("en", "SCB Tables") },
CatalogDescriptions = new List<KeyValuePair<string, string>> { new KeyValuePair<string, string>("sv", "SCB - Beskrivning"), new KeyValuePair<string, string>("en", "SCB - Description") },
PublisherName = "Statistics Sweden",
DatabaseId = @"C:\Temp\Databases\Example\Menu.xml",
DatabaseType = DatabaseType.PX,
//DBid = @"C:\Temp\StatFin2018\StatFin\Menu.xml",
//Fetcher = new PXFetcher(@"C:\Temp\StatFin2018"),
PublisherNames = new List<KeyValuePair<string, string>> { new KeyValuePair<string, string>("en", "Statistics Sweden"), new KeyValuePair<string, string>("sv", "SCB") },
DatabaseId = @"C:\Temp\Databases\Example/Menu.xml",
//DatabaseId = @"C:\Temp\StatFin2018\StatFin\Menu.xml",
//DatabaseId = "ssd",
DatabaseType = DatabaseType.PX,
//DatabaseType = DatabaseType.CNMM,
LandingPageUrl = "http://www.statistikdatabasen.scb.se/goto/",
LandingPageUrl = "http://localhost:56338/goto/",
//LandingPageUrl = "http://www.statistikdatabasen.scb.se/goto/",
License = "http://creativecommons.org/publicdomain/zero/1.0/",
ThemeMapping = @"C:\Temp\DataportalXML\Themes.json",
OrganizationMapping = @"C:\Temp\DataportalXML\Organizations.json",
ThemeMapping = @"C:\Temp\DataportalXML\TestApp\Themes.json",
OrganizationMapping = @"C:\Temp\DataportalXML\TestApp\Organizations.json",
MainLanguage = "en",
};
DcatWriter.WriteToFile("../../../test.xml", settings);
Expand Down

0 comments on commit 83c077d

Please sign in to comment.