Skip to content

Commit

Permalink
Organize BasicDatasets into folders (#518)
Browse files Browse the repository at this point in the history
This divides them into the folders:
- cv
- cv.classification
- nlp
- tabular

Change-Id: Ic925c69caae628d95d33f010e62c14bdb5ae6d17
  • Loading branch information
zachgk authored Jan 13, 2021
1 parent 5b59336 commit 79fa9ac
Show file tree
Hide file tree
Showing 60 changed files with 234 additions and 49 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv;

import ai.djl.Application;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.modality.cv.transform.ToTensor;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv;

import ai.djl.Application.CV;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.modality.cv.transform.ToTensor;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv;

import com.google.gson.annotations.SerializedName;
import java.util.List;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv;

import ai.djl.util.JsonUtils;
import java.io.IOException;
Expand Down Expand Up @@ -121,11 +121,10 @@ public List<Long> getAnnotationIdByImageId(long imageId) {
}

/**
* Returns an {@link ai.djl.basicdataset.CocoMetadata.Annotation} that corresponds to a given
* annotation id.
* Returns an {@link CocoMetadata.Annotation} that corresponds to a given annotation id.
*
* @param annotationId the annotation id to retrieve an annotation for
* @return an {@link ai.djl.basicdataset.CocoMetadata.Annotation}
* @return an {@link CocoMetadata.Annotation}
*/
public CocoMetadata.Annotation getAnnotationById(long annotationId) {
return annotationMap.get(annotationId);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv;

import ai.djl.Application.CV;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.modality.cv.transform.ToTensor;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv.classification;

import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
Expand Down Expand Up @@ -140,7 +140,6 @@ public abstract static class ImageFolderBuilder<T extends ImageFolderBuilder<T>>
int maxDepth;

protected ImageFolderBuilder() {
flag = Image.Flag.COLOR;
maxDepth = 1;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv.classification;

import ai.djl.Application.CV;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.modality.cv.Image;
import ai.djl.modality.cv.ImageFactory;
import ai.djl.modality.cv.transform.ToTensor;
Expand Down Expand Up @@ -145,7 +146,7 @@ private String getUsagePath() {
}
}

/** A builder for a {@link ai.djl.basicdataset.CaptchaDataset}. */
/** A builder for a {@link CaptchaDataset}. */
public static final class Builder extends BaseBuilder<Builder> {

Repository repository;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv.classification;

import ai.djl.Application.CV;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.engine.Engine;
import ai.djl.modality.cv.transform.ToTensor;
import ai.djl.ndarray.NDArray;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv.classification;

import ai.djl.Application.CV;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.engine.Engine;
import ai.djl.modality.cv.transform.ToTensor;
import ai.djl.ndarray.NDArray;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
* with the License. A copy of the License is located at
*
* http://aws.amazon.com/apache2.0/
*
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset.cv.classification;

import ai.djl.modality.cv.Image;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.training.dataset.RandomAccessDataset;
import ai.djl.training.dataset.Record;
import java.io.IOException;

/**
* A helper to create {@link ai.djl.training.dataset.Dataset}s for {@link
* ai.djl.Application.CV#IMAGE_CLASSIFICATION}.
*/
public abstract class ImageClassificationDataset extends RandomAccessDataset {

Image.Flag flag;

/**
* Creates a new instance of {@link RandomAccessDataset} with the given necessary
* configurations.
*
* @param builder a builder with the necessary configurations
*/
public ImageClassificationDataset(BaseBuilder<?> builder) {
super(builder);
this.flag = builder.flag;
}

/**
* Returns the image at the given index in the dataset.
*
* @param index the index (if the dataset is a list of data items)
* @return the image
* @throws IOException if the image could not be loaded
*/
protected abstract Image getImage(long index) throws IOException;

/**
* Returns the class of the data item at the given index.
*
* @param index the index (if the dataset is a list of data items)
* @return the class number or the index into the list of classes of the desired class name
* @throws IOException if the data could not be loaded
*/
protected abstract long getClassNumber(long index) throws IOException;

/** {@inheritDoc} */
@Override
public Record get(NDManager manager, long index) throws IOException {
NDList data = new NDList(getImage(index).toNDArray(manager, flag));
NDList label = new NDList(manager.create(getClassNumber(index)));
return new Record(data, label);
}

/**
* Used to build an {@link ImageClassificationDataset}.
*
* @param <T> the builder type
*/
@SuppressWarnings("rawtypes")
public abstract static class BaseBuilder<T extends BaseBuilder<T>>
extends RandomAccessDataset.BaseBuilder<T> {

Image.Flag flag;

protected BaseBuilder() {
flag = Image.Flag.COLOR;
}

/**
* Sets the optional color mode flag.
*
* @param flag the color mode flag
* @return this builder
*/
public T optFlag(Image.Flag flag) {
this.flag = flag;
return self();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv.classification;

import ai.djl.modality.cv.transform.ToTensor;
import ai.djl.translate.Pipeline;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv.classification;

import ai.djl.modality.cv.transform.ToTensor;
import ai.djl.training.dataset.Dataset;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.cv.classification;

import ai.djl.Application.CV;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.engine.Engine;
import ai.djl.modality.cv.transform.ToTensor;
import ai.djl.ndarray.NDArray;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/*
* Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
* with the License. A copy of the License is located at
*
* http://aws.amazon.com/apache2.0/
*
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/

/**
* Contains a library of built-in datasets for {@link ai.djl.Application.CV#IMAGE_CLASSIFICATION}.
*/
package ai.djl.basicdataset.cv.classification;
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
/*
* Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
* with the License. A copy of the License is located at
*
* http://aws.amazon.com/apache2.0/
*
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/

/** Contains a library of built-in datasets for {@link ai.djl.Application.CV}. */
package ai.djl.basicdataset.cv;
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.nlp;

import ai.djl.Application;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.basicdataset.tabular.CsvDataset;
import ai.djl.repository.Artifact;
import ai.djl.repository.MRL;
import ai.djl.repository.Repository;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.nlp;

import ai.djl.Application.NLP;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.basicdataset.RawDataset;
import ai.djl.ndarray.NDManager;
import ai.djl.repository.Artifact;
import ai.djl.repository.MRL;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.nlp;

import ai.djl.Application.NLP;
import ai.djl.modality.nlp.embedding.EmbeddingException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.nlp;

import ai.djl.Application.NLP;
import ai.djl.modality.nlp.embedding.EmbeddingException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.nlp;

import ai.djl.basicdataset.BasicDatasets;
import ai.djl.basicdataset.utils.TextData;
import ai.djl.basicdataset.utils.TextData.Configuration;
import ai.djl.engine.Engine;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
/*
* Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
* with the License. A copy of the License is located at
*
* http://aws.amazon.com/apache2.0/
*
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/

/** Contains a library of built-in datasets for {@link ai.djl.Application.NLP}. */
package ai.djl.basicdataset.nlp;
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.tabular;

import ai.djl.Application;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.Shape;
Expand Down Expand Up @@ -54,8 +55,7 @@ public final class AirfoilRandomAccess extends CsvDataset {
private Map<String, Float> std;

/**
* Creates an instance of {@code RandomAccessDataset} with the arguments in {@link
* FashionMnist.Builder}.
* Creates an instance of {@code RandomAccessDataset} with the arguments in {@link Builder}.
*
* @param builder a builder with the required arguments
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
* OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package ai.djl.basicdataset;
package ai.djl.basicdataset.tabular;

import ai.djl.Application.Tabular;
import ai.djl.basicdataset.BasicDatasets;
import ai.djl.repository.Artifact;
import ai.djl.repository.MRL;
import ai.djl.repository.Repository;
Expand Down
Loading

0 comments on commit 79fa9ac

Please sign in to comment.