Skip to content

Commit

Permalink
Move version into MRL (#1114)
Browse files Browse the repository at this point in the history 
Change-Id: Iafa45f59cf966d8134b2cc127d1243767513132b
  • Loading branch information
@frankfliu
frankfliu authored Jul 20, 2021
1 parent 00ac45c commit 154ae6e
Show file tree
Hide file tree
Showing 42 changed files with 481 additions and 413 deletions.
6 changes: 3 additions & 3 deletions api/src/main/java/ai/djl/repository/JarRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public Metadata locate(MRL mrl) {

/** {@inheritDoc} */
@Override
public Artifact resolve(MRL mrl, String version, Map<String, String> filter) {
public Artifact resolve(MRL mrl, Map<String, String> filter) {
List<Artifact> artifacts = locate(mrl).getArtifacts();
if (artifacts.isEmpty()) {
return null;
Expand All @@ -89,7 +89,7 @@ public Artifact resolve(MRL mrl, String version, Map<String, String> filter) {
public List<MRL> getResources() {
Metadata m = getMetadata();
if (m != null && !m.getArtifacts().isEmpty()) {
MRL mrl = MRL.undefined(m.getGroupId(), m.getArtifactId());
MRL mrl = MRL.undefined(this, m.getGroupId(), m.getArtifactId());
return Collections.singletonList(mrl);
}
return Collections.emptyList();
Expand Down Expand Up @@ -128,7 +128,7 @@ private synchronized Metadata getMetadata() {
metadata.setArtifactId(artifactId);
metadata.setArtifacts(Collections.singletonList(artifact));
String hash = md5hash(uri.toString());
MRL mrl = MRL.model(Application.UNDEFINED, DefaultModelZoo.GROUP_ID, hash);
MRL mrl = model(Application.UNDEFINED, DefaultModelZoo.GROUP_ID, hash);
metadata.setRepositoryUri(mrl.toURI());

return metadata;
Expand Down
9 changes: 4 additions & 5 deletions api/src/main/java/ai/djl/repository/LocalRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,9 @@ public Metadata locate(MRL mrl) throws IOException {

/** {@inheritDoc} */
@Override
public Artifact resolve(MRL mrl, String version, Map<String, String> filter)
throws IOException {
public Artifact resolve(MRL mrl, Map<String, String> filter) throws IOException {
Metadata metadata = locate(mrl);
VersionRange range = VersionRange.parse(version);
VersionRange range = VersionRange.parse(mrl.getVersion());
List<Artifact> artifacts = metadata.search(range, filter);
if (artifacts.isEmpty()) {
return null;
Expand All @@ -117,9 +116,9 @@ public List<MRL> getResources() {
String groupId = metadata.getGroupId();
String artifactId = metadata.getArtifactId();
if ("dataset".equals(type)) {
list.add(MRL.dataset(application, groupId, artifactId));
list.add(dataset(application, groupId, artifactId));
} else if ("model".equals(type)) {
list.add(MRL.model(application, groupId, artifactId));
list.add(model(application, groupId, artifactId));
}
} catch (IOException e) {
logger.warn("Failed to read metadata.json", e);
Expand Down
153 changes: 144 additions & 9 deletions api/src/main/java/ai/djl/repository/MRL.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,13 @@
package ai.djl.repository;

import ai.djl.Application;
import ai.djl.util.Progress;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* The {@code MRL} (Machine learning Resource Locator) is a pointer to a {@link Metadata} "resource"
Expand All @@ -36,59 +42,89 @@
*/
public final class MRL {

private static final Logger logger = LoggerFactory.getLogger(MRL.class);

private String type;
private Application application;
private String groupId;
private String artifactId;
private String version;
private Repository repository;
private Metadata metadata;

/**
* Constructs an MRL.
*
* @param repository the {@link Repository}
* @param type the resource type
* @param application the resource application
* @param groupId the desired groupId
* @param artifactId the desired artifactId
* @param version the resource version
*/
private MRL(String type, Application application, String groupId, String artifactId) {
private MRL(
Repository repository,
String type,
Application application,
String groupId,
String artifactId,
String version) {
this.repository = repository;
this.type = type;
this.application = application;
this.groupId = groupId;
this.artifactId = artifactId;
this.version = version;
}

/**
* Creates a model {@code MRL} with specified application.
*
* @param repository the {@link Repository}
* @param application the desired application
* @param groupId the desired groupId
* @param artifactId the desired artifactId
* @param version the resource version
* @return a model {@code MRL}
*/
public static MRL model(Application application, String groupId, String artifactId) {
return new MRL("model", application, groupId, artifactId);
public static MRL model(
Repository repository,
Application application,
String groupId,
String artifactId,
String version) {
return new MRL(repository, "model", application, groupId, artifactId, version);
}

/**
* Creates a dataset {@code MRL} with specified application.
*
* @param repository the {@link Repository}
* @param application the desired application
* @param groupId the desired groupId
* @param artifactId the desired artifactId
* @param version the resource version
* @return a dataset {@code MRL}
*/
public static MRL dataset(Application application, String groupId, String artifactId) {
return new MRL("dataset", application, groupId, artifactId);
public static MRL dataset(
Repository repository,
Application application,
String groupId,
String artifactId,
String version) {
return new MRL(repository, "dataset", application, groupId, artifactId, version);
}

/**
* Creates a dataset {@code MRL} with specified application.
*
* @param repository the {@link Repository}
* @param groupId the desired groupId
* @param artifactId the desired artifactId
* @return a dataset {@code MRL}
*/
public static MRL undefined(String groupId, String artifactId) {
return new MRL("", Application.UNDEFINED, groupId, artifactId);
public static MRL undefined(Repository repository, String groupId, String artifactId) {
return new MRL(repository, "", Application.UNDEFINED, groupId, artifactId, null);
}

/**
Expand All @@ -112,9 +148,18 @@ public URI toURI() {
}

/**
* Returns the resource application.
* Returns the repository.
*
* @return the repository
*/
public Repository getRepository() {
return repository;
}

/**
* Returns the application.
*
* @return the resource application
* @return the application
*/
public Application getApplication() {
return application;
Expand All @@ -138,6 +183,96 @@ public String getArtifactId() {
return artifactId;
}

/**
* Returns the version.
*
* @return the version
*/
public String getVersion() {
return version;
}

/**
* Returns the default artifact.
*
* @return the default artifact
* @throws IOException for various exceptions depending on the specific dataset
*/
public Artifact getDefaultArtifact() throws IOException {
return repository.resolve(this, null);
}

/**
* Returns the first artifact that matches a given criteria.
*
* @param criteria the criteria to match against
* @return the first artifact that matches the criteria. Null will be returned if no artifact
* matches
* @throws IOException for errors while loading the model
*/
public Artifact match(Map<String, String> criteria) throws IOException {
List<Artifact> list = search(criteria);
if (list.isEmpty()) {
return null;
}
return list.get(0);
}

/**
* Returns a list of artifacts in this resource.
*
* @return a list of artifacts in this resource
* @throws IOException for errors while loading the model
*/
public List<Artifact> listArtifacts() throws IOException {
return getMetadata().getArtifacts();
}

/**
* Prepares the artifact for use.
*
* @param artifact the artifact to prepare
* @throws IOException if it failed to prepare
*/
public void prepare(Artifact artifact) throws IOException {
prepare(artifact, null);
}

/**
* Prepares the artifact for use with progress tracking.
*
* @param artifact the artifact to prepare
* @param progress the progress tracker
* @throws IOException if it failed to prepare
*/
public void prepare(Artifact artifact, Progress progress) throws IOException {
if (artifact != null) {
logger.debug("Preparing artifact: {}, {}", repository.getName(), artifact);
repository.prepare(artifact, progress);
}
}

/**
* Returns all the artifacts that match a given criteria.
*
* @param criteria the criteria to match against
* @return all the artifacts that match a given criteria
* @throws IOException for errors while loading the model
*/
private List<Artifact> search(Map<String, String> criteria) throws IOException {
return getMetadata().search(VersionRange.parse(version), criteria);
}

private Metadata getMetadata() throws IOException {
if (metadata == null) {
metadata = repository.locate(this);
if (metadata == null) {
throw new IOException(this + " resource not found.");
}
}
return metadata;
}

/** {@inheritDoc} */
@Override
public String toString() {
Expand Down
5 changes: 2 additions & 3 deletions api/src/main/java/ai/djl/repository/RemoteRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,9 @@ public Metadata locate(MRL mrl) throws IOException {

/** {@inheritDoc} */
@Override
public Artifact resolve(MRL mrl, String version, Map<String, String> filter)
throws IOException {
public Artifact resolve(MRL mrl, Map<String, String> filter) throws IOException {
Metadata metadata = locate(mrl);
VersionRange range = VersionRange.parse(version);
VersionRange range = VersionRange.parse(mrl.getVersion());
List<Artifact> artifacts = metadata.search(range, filter);
if (artifacts.isEmpty()) {
return null;
Expand Down
54 changes: 52 additions & 2 deletions api/src/main/java/ai/djl/repository/Repository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,57 @@ static void registerRepositoryFactory(RepositoryFactory factory) {
RepositoryFactoryImpl.registerRepositoryFactory(factory);
}

/**
* Creates a model {@code MRL} with specified application.
*
* @param application the desired application
* @param groupId the desired groupId
* @param artifactId the desired artifactId
* @return a model {@code MRL}
*/
default MRL model(Application application, String groupId, String artifactId) {
return model(application, groupId, artifactId, null);
}

/**
* Creates a model {@code MRL} with specified application.
*
* @param application the desired application
* @param groupId the desired groupId
* @param artifactId the desired artifactId
* @param version the resource version
* @return a model {@code MRL}
*/
default MRL model(Application application, String groupId, String artifactId, String version) {
return MRL.model(this, application, groupId, artifactId, version);
}

/**
* Creates a dataset {@code MRL} with specified application.
*
* @param application the desired application
* @param groupId the desired groupId
* @param artifactId the desired artifactId
* @return a dataset {@code MRL}
*/
default MRL dataset(Application application, String groupId, String artifactId) {
return dataset(application, groupId, artifactId, null);
}

/**
* Creates a dataset {@code MRL} with specified application.
*
* @param application the desired application
* @param groupId the desired groupId
* @param artifactId the desired artifactId
* @param version the resource version
* @return a dataset {@code MRL}
*/
default MRL dataset(
Application application, String groupId, String artifactId, String version) {
return MRL.dataset(this, application, groupId, artifactId, version);
}

/**
* Returns whether the repository is remote repository.
*
Expand Down Expand Up @@ -132,12 +183,11 @@ static void registerRepositoryFactory(RepositoryFactory factory) {
* Returns the artifact matching a mrl, version, and property filter.
*
* @param mrl the mrl to match the artifact against
* @param version the version of the artifact
* @param filter the property filter
* @return the matched artifact
* @throws IOException if it failed to load the artifact
*/
Artifact resolve(MRL mrl, String version, Map<String, String> filter) throws IOException;
Artifact resolve(MRL mrl, Map<String, String> filter) throws IOException;

/**
* Returns an {@link InputStream} for an item in a repository.
Expand Down
Loading

0 comments on commit 154ae6e

Please sign in to comment.