add vlt source code

indexr-codegen/pom.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>indexr</artifactId>
+        <groupId>io.indexr</groupId>
+        <version>0.5.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>indexr-codegen</artifactId>
+
+    <name>${project.groupId}:${project.artifactId}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.indexr</groupId>
+            <artifactId>indexr-segment</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+
+</project>

indexr-hive/pom.xml

@@ -46,6 +46,11 @@
             <artifactId>indexr-segment</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.indexr</groupId>
+            <artifactId>indexr-vlt-segment</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-exec</artifactId>

indexr-server/pom.xml

@@ -47,11 +47,11 @@
             <artifactId>indexr-segment</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <!--<dependency>-->
-        <!--<groupId>org.eclipse.jetty</groupId>-->
-        <!--<artifactId>jetty-server</artifactId>-->
-        <!--<version>9.3.11.v20160721</version>-->
-        <!--</dependency>-->
+        <dependency>
+            <groupId>io.indexr</groupId>
+            <artifactId>indexr-vlt-segment</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka_2.10</artifactId>

indexr-vlt-segment/pom.xml

@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>indexr</artifactId>
+        <groupId>io.indexr</groupId>
+        <version>0.5.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>indexr-vlt-segment</artifactId>
+
+    <name>${project.groupId}:${project.artifactId}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.indexr</groupId>
+            <artifactId>indexr-segment</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.indexr</groupId>
+            <artifactId>indexr-segment</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>it.unimi.dsi</groupId>
+            <artifactId>fastutil</artifactId>
+            <version>6.5.9</version>
+        </dependency>
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-enforcer-plugin</artifactId>
+                <version>1.4.1</version>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>2.6</version>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>1.2.1</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>io.indexr</groupId>
+                        <artifactId>indexr-codegen</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                </dependencies>
+                <executions>
+                    <execution>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>java</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <mainClass>io.indexr.vlt.codec.pack.PackingClassGen</mainClass>
+                    <includePluginDependencies>true</includePluginDependencies>
+                    <arguments>
+                        <argument>${basedir}/target/generated-sources</argument>
+                    </arguments>
+                    <sourceRoot>${basedir}/target/generated-sources</sourceRoot>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

indexr-vlt-segment/src/main/java/io/indexr/plugin/VLTPlugin.java

@@ -0,0 +1,15 @@
+package io.indexr.plugin;
+
+import io.indexr.vlt.segment.VersionAdapter_VLT;
+
+import io.indexr.segment.SegmentMode;
+
+public class VLTPlugin {
+    public static SegmentMode VLT = new SegmentMode(10001, "vlt", new VersionAdapter_VLT());
+
+    @InitPlugin
+    public static void init() {
+        SegmentMode.addMode(VLT);
+        SegmentMode.setDefault(VLT);
+    }
+}

indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/Codec.java

@@ -0,0 +1,28 @@
+package io.indexr.vlt.codec;
+
+public interface Codec extends Encoder, Decoder {
+
+    CodecType type();
+
+    static int outputBufferSize(CodecType type, int valCount, int dataSize) {
+        int size;
+        switch (type) {
+            case SIMPLE_DICT:
+            case DICT_COMPRESS:
+                size = dataSize + (valCount << 2) + 1024;
+                break;
+            case DICT:
+                //size = dataSize + (valCount << 2) + 1024 + (1 << 17) + (1 << 16);
+                size = dataSize + (valCount << 2) + 1024;
+                break;
+            default:
+                if (valCount <= 64) {
+                    size = dataSize << 1;
+                } else {
+                    size = (int) (dataSize * 1.5);
+                }
+        }
+        // minimal 1024 bytes.
+        return Math.max(1 << 10, size);
+    }
+}

indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/CodecType.java

@@ -0,0 +1,28 @@
+package io.indexr.vlt.codec;
+
+public enum CodecType {
+    PLAIN(1),
+    DELTA(2),
+    RLE(3),
+    SIMPLE_DICT(4),
+    /** This codec is private used only */
+    DICT(5),
+    DICT_COMPRESS(6),
+    //
+    ;
+
+    public final int id;
+
+    CodecType(int id) {
+        this.id = id;
+    }
+
+    public static CodecType fromId(int id) {
+        for (CodecType t : values()) {
+            if (t.id == id) {
+                return t;
+            }
+        }
+        throw new RuntimeException("Illegal codec id: " + id);
+    }
+}

indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/Decoder.java

@@ -0,0 +1,25 @@
+package io.indexr.vlt.codec;
+
+// Returns how many bytes it consumed. Which should be equal to the return value of Encoder,
+// otherwise, you caught a bug.
+public interface Decoder {
+    default int decodeInts(int valCount, long inputAddr, long outputAddr, int outputLimit) {
+        throw new UnsupportedOperationException();
+    }
+
+    default int decodeLongs(int valCount, long inputAddr, long outputAddr, int outputLimit) {
+        throw new UnsupportedOperationException();
+    }
+
+    default int decodeFloats(int valCount, long inputAddr, long outputAddr, int outputLimit) {
+        throw new UnsupportedOperationException();
+    }
+
+    default int decodeDoubles(int valCount, long inputAddr, long outputAddr, int ouputLimit) {
+        throw new UnsupportedOperationException();
+    }
+
+    default int decodeStrings(int valCount, long inputAddr, long outputAddr, int ouputLimit) {
+        throw new UnsupportedOperationException();
+    }
+}

indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/DecoderItf.java

@@ -0,0 +1,5 @@
+package io.indexr.vlt.codec;
+
+public interface DecoderItf {
+    int decode(int valCount, long inputAddr, long outputAddr, int outputLimit);
+}

indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/Encoder.java

@@ -0,0 +1,49 @@
+package io.indexr.vlt.codec;
+
+import io.indexr.util.Wrapper;
+
+/**
+ * Returns the encoded data size.
+ */
+public interface Encoder {
+    default int encodeInts(int valCount, long inputAddr, long outputAddr, int outputLimit) {
+        return encodeInts(valCount, inputAddr, outputAddr, outputLimit, new Wrapper());
+    }
+
+    default int encodeInts(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo) {
+        throw new UnsupportedOperationException();
+    }
+
+    default int encodeLongs(int valCount, long inputAddr, long outputAddr, int outputLimit) {
+        return encodeLongs(valCount, inputAddr, outputAddr, outputLimit, new Wrapper());
+    }
+
+    default int encodeLongs(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo) {
+        throw new UnsupportedOperationException();
+    }
+
+    default int encodeFloats(int valCount, long inputAddr, long outputAddr, int outputLimit) {
+        return encodeFloats(valCount, inputAddr, outputAddr, outputLimit, new Wrapper());
+    }
+
+    default int encodeFloats(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo) {
+        throw new UnsupportedOperationException();
+    }
+
+    default int encodeDoubles(int valCount, long inputAddr, long outputAddr, int outputLimit) {
+        return encodeDoubles(valCount, inputAddr, outputAddr, outputLimit, new Wrapper());
+    }
+
+    default int encodeDoubles(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo) {
+        throw new UnsupportedOperationException();
+    }
+
+    default int encodeStrings(int valCount, long inputAddr, long outputAddr, int outputLimit) {
+        return encodeStrings(valCount, inputAddr, outputAddr, outputLimit, new Wrapper());
+    }
+
+    default int encodeStrings(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo) {
+        throw new UnsupportedOperationException();
+    }
+
+}

indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/EncoderItf.java

@@ -0,0 +1,7 @@
+package io.indexr.vlt.codec;
+
+import io.indexr.util.Wrapper;
+
+public interface EncoderItf {
+    int encode(int valCount, long inputAddr, long outputAddr, int outputLimit, Wrapper extraInfo);
+}

indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/Encoding.java

@@ -0,0 +1,4 @@
+package io.indexr.vlt.codec;
+
+public enum Encoding {
+}

indexr-vlt-segment/src/main/java/io/indexr/vlt/codec/ErrorCode.java

@@ -0,0 +1,9 @@
+package io.indexr.vlt.codec;
+
+public class ErrorCode {
+    public static final int BUFFER_OVERFLOW = -1;
+    public static final int DATA_CORRUPTED = -2;
+    public static final int DATA_IMPROPER = -3;
+    public static final int BUG = -100;
+
+}