Koios 1.2.0a alpha (#298)

## Description
<!--- Describe your changes -->

To-Do:
- [x] update epoch_params section to include new governance params
- [x] Add `deposit` field to `account_info`  (could park to future patch)
- [x] Add `delegated_drep` field to `account_info`
- [x] Introduce governance section (add compatibility with BF endpoints where possible for redundancy from lib providers pov)
  - [x] /committee_info
  - [x] /committee_votes
  - [x] /drep_list
  - [x] /drep_delegators
  - [x] /drep_info
  - [x] /drep_metadata
  - [x] /drep_updates
  - [x] /drep_votes
  - [x] /drep_epoch_summary
  - [x] /proposal_list
  - [x] /proposal_votes
  - [x] /pool_votes
- [x] Extend existing endpoints to include governance information (could park to future patch):
  - [x] /tx_info
- [x] Make `bytecode` optional in tx_info, closes #293
- [x] Update `tx_info` to include various toggles in input (similar to `block_tx_info` , and in addition `_bytecode`)
- [x] Update `block_tx_info` to include `_bytecode` toggle
- [x] Update `block_tx_info` and `tx_info` to include `treasury_donation` field
- [x] Add `tx_cbor` endpoint
- [ ] Add `tx_cbor2json` endpoint (could park to future patch)
- [x] Change how pool metadata is handled. Remove all metadata fields from pool_info_cache and query data live in endpoints
- [x] Add Bech32 decode/encoder PG extension
- [ ] Implement [CIP-129](https://github.com/cardano-foundation/CIPs/pull/857/files) (Dependent on PG B32 extension), utility and extensions are done , but their usage in endpoints parked for next [beta] release)

---------

Co-authored-by: Ola <ola.ahlman@gmail.com>
Co-authored-by: hodlonaut <greg@_HOSTNAME_>
Co-authored-by: Greg B <adahoskinson@gmail.com>
Co-authored-by: Greg Beresnev <greg.beresnev@yarris.com>
Co-authored-by: KoT_B_KocMoce <49576827+hodlonaut@users.noreply.github.com>

files/grest/rpc/01_cached_tables/pool_info_cache.sql

@@ -14,6 +14,7 @@ CREATE TABLE grest.pool_info_cache (
   margin double precision NOT NULL,
   fixed_cost lovelace NOT NULL,
   pledge lovelace NOT NULL,
+  deposit lovelace,
   reward_addr character varying,
   owners character varying [],
   relays jsonb [],
@@ -35,6 +36,7 @@ CREATE OR REPLACE FUNCTION grest.pool_info_insert(
   _margin double precision,
   _fixed_cost lovelace,
   _pledge lovelace,
+  _deposit lovelace,
   _reward_addr_id bigint,
   _meta_id bigint
 )
@@ -63,6 +65,8 @@ BEGIN
     _pool_status := 'retired';
   END IF;
 
+  DELETE FROM grest.pool_info_cache WHERE pool_hash_id = _hash_id;
+
   INSERT INTO grest.pool_info_cache (
     tx_id,
     update_id,
@@ -76,6 +80,7 @@ BEGIN
     margin,
     fixed_cost,
     pledge,
+    deposit,
     reward_addr,
     owners,
     relays,
@@ -98,6 +103,7 @@ BEGIN
       _margin,
       _fixed_cost,
       _pledge,
+      _deposit,
       sa.view,
       ARRAY(
         SELECT sa.view
@@ -184,8 +190,8 @@ BEGIN
 
   UPDATE grest.pool_info_cache
   SET
-      pool_status = _pool_status,
-      retiring_epoch = _retiring_epoch
+    pool_status = _pool_status,
+    retiring_epoch = _retiring_epoch
   WHERE pool_hash_id = _pool_hash_id
     AND tx_id = _latest_pool_update_tx_id;
 
@@ -201,9 +207,20 @@ LANGUAGE plpgsql
 AS $$
 DECLARE
   _latest_pool_update_id integer;
+  _calc_meta_id bigint;
 BEGIN
   IF (tg_table_name = 'pool_update') THEN
     IF (tg_op = 'INSERT') THEN
+
+      -- for url/hash of most recent update, find last off_chain data record
+      SELECT coalesce(max(ocpd.pmr_id), new.meta_id) INTO _calc_meta_id
+      FROM off_chain_pool_data ocpd 
+      INNER JOIN pool_metadata_ref pmr ON ocpd.pmr_id = pmr.id 
+      INNER JOIN pool_metadata_ref pmr2 ON pmr2.id = new.meta_id AND
+        pmr2.pool_id = pmr.pool_id AND
+        pmr2.url = pmr.url AND
+        pmr2.hash = pmr.hash;
+
       PERFORM grest.pool_info_insert(
         new.id,
         new.registered_tx_id,
@@ -213,8 +230,9 @@ BEGIN
         new.margin,
         new.fixed_cost,
         new.pledge,
+        new.deposit,
         new.reward_addr_id,
-        new.meta_id
+        _calc_meta_id
       );
     ELSIF (tg_op = 'DELETE') THEN
       DELETE FROM grest.pool_info_cache
@@ -297,13 +315,24 @@ FOR EACH ROW EXECUTE FUNCTION grest.pool_info_retire_status();
 DO $$
 DECLARE
   _latest_pool_info_tx_id bigint;
+  _calc_meta_id bigint;
   rec RECORD;
 BEGIN
   SELECT COALESCE(MAX(tx_id), 0) INTO _latest_pool_info_tx_id FROM grest.pool_info_cache;
 
   FOR rec IN (
     SELECT * FROM public.pool_update AS pu WHERE pu.registered_tx_id > _latest_pool_info_tx_id
   ) LOOP
+
+    -- for url/hash of most recent update, find last off_chain data record
+    SELECT coalesce(max(ocpd.pmr_id), rec.meta_id) INTO _calc_meta_id
+    FROM off_chain_pool_data ocpd 
+    INNER JOIN pool_metadata_ref pmr ON ocpd.pmr_id = pmr.id 
+    INNER JOIN pool_metadata_ref pmr2 ON pmr2.id = rec.meta_id AND
+      pmr2.pool_id = pmr.pool_id AND
+      pmr2.url = pmr.url AND
+      pmr2.hash = pmr.hash;
+
     PERFORM grest.pool_info_insert(
       rec.id,
       rec.registered_tx_id,
@@ -313,8 +342,9 @@ BEGIN
       rec.margin,
       rec.fixed_cost,
       rec.pledge,
+      rec.deposit,
       rec.reward_addr_id,
-      rec.meta_id
+      _calc_meta_id
     );
   END LOOP;
 

files/grest/rpc/02_indexes/13_3_00.sql

@@ -1,2 +1,3 @@
 CREATE INDEX IF NOT EXISTS pool_stat_pool_hash_id ON pool_stat(pool_hash_id);
-CREATE INDEX IF NOT EXISTS pool_stat_epoch_no ON pool_stat(epoch_no);
+CREATE INDEX IF NOT EXISTS pool_stat_epoch_no ON pool_stat(epoch_no);
+CREATE INDEX IF NOT EXISTS idx_drep_hash_view ON drep_hash (view);

files/grest/rpc/03_utilities/cip129.sql

@@ -0,0 +1,107 @@
+-- Binary format
+--     1 byte     variable length
+--      <------> <------------------->
+--     ┌────────┬─────────────────────┐
+--     │ header │        key      │
+--     └────────┴─────────────────────┘
+--         🔎
+--         ╎          7 6 5 4 3 2 1 0
+--         ╎         ┌─┬─┬─┬─┬─┬─┬─┬─┐
+--         ╰╌╌╌╌╌╌╌╌ |t│t│t│t│c│c│c│c│
+--                   └─┴─┴─┴─┴─┴─┴─┴─┘
+--
+-- Key Type (`t t t t . . . .`)          | Key
+-- ---                                   | ---
+-- `0000....`                            | CC Hot
+-- `0001....`                            | CC Cold
+-- `0010....`                            | DRep
+--
+-- Credential Type (`. . . . c c c c`)   | Semantic
+-- ---                                   | ---
+-- `....0010`                            | Key Hash
+-- `....0011`                            | Script Hash
+
+CREATE OR REPLACE FUNCTION grest.cip129_cc_hot_to_hex(_cc_hot text)
+RETURNS bytea
+LANGUAGE plpgsql STABLE
+AS $$
+BEGIN
+  IF LENGTH(_cc_hot) = 60 THEN
+    RETURN substring(b32_decode(_cc_hot) from 2);
+  ELSE
+    RETURN b32_decode(_cc_hot);
+  END IF;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION grest.cip129_hex_to_cc_hot(_raw bytea, _is_script boolean)
+RETURNS text
+LANGUAGE plpgsql STABLE
+AS $$
+BEGIN
+  IF _is_script THEN
+    RETURN b32_encode('cc_hot', ('\x03'::bytea || _raw)::text);
+  ELSE
+    RETURN b32_encode('cc_hot', ('\x02'::bytea || _raw)::text);
+  END IF;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION grest.cip129_cc_cold_to_hex(_cc_cold text)
+RETURNS bytea
+LANGUAGE plpgsql STABLE
+AS $$
+BEGIN
+  IF LENGTH(_cc_cold) = 61 THEN
+    RETURN substring(b32_decode(_cc_cold) from 2);
+  ELSE
+    RETURN b32_decode(_cc_cold);
+  END IF;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION grest.cip129_hex_to_cc_cold(_raw bytea, _is_script boolean)
+RETURNS text
+LANGUAGE plpgsql STABLE
+AS $$
+BEGIN
+  IF _is_script THEN
+    RETURN b32_encode('cc_cold', ('\x13'::bytea || _raw)::text);
+  ELSE
+    RETURN b32_encode('cc_cold', ('\x12'::bytea || _raw)::text);
+  END IF;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION grest.cip129_drep_id_to_hex(_drep_id text)
+RETURNS bytea
+LANGUAGE plpgsql STABLE
+AS $$
+BEGIN
+  IF LENGTH(_drep_id) = 58 THEN
+    RETURN substring(b32_decode(_drep_id) from 2);
+  ELSE
+    RETURN b32_decode(_drep_id);
+  END IF;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION grest.cip129_hex_to_drep_id(_raw bytea, _is_script boolean)
+RETURNS text
+LANGUAGE plpgsql STABLE
+AS $$
+BEGIN
+  IF _is_script THEN
+    RETURN b32_encode('drep', ('\x23'::bytea || _raw)::text);
+  ELSE
+    RETURN b32_encode('drep', ('\x22'::bytea || _raw)::text);
+  END IF;
+END;
+$$;
+
+COMMENT ON FUNCTION grest.cip129_cc_hot_to_hex IS 'Returns binary hex from Constitutional Committee Hot Credential ID in old or new (CIP-129) format'; -- noqa: LT01
+COMMENT ON FUNCTION grest.cip129_hex_to_cc_hot IS 'Returns Constitutional Committee Hot Credential ID in CIP-129 format from raw binary hex'; -- noqa: LT01
+COMMENT ON FUNCTION grest.cip129_cc_cold_to_hex IS 'Returns binary hex from Constitutional Committee Cold Credential ID in old or new (CIP-129) format'; -- noqa: LT01
+COMMENT ON FUNCTION grest.cip129_hex_to_cc_cold IS 'Returns Constitutional Committee Cold Credential ID in CIP-129 format from raw binary hex'; -- noqa: LT01
+COMMENT ON FUNCTION grest.cip129_drep_id_to_hex IS 'Returns binary hex from DRep Credential ID in old or new (CIP-129) format'; -- noqa: LT01
+COMMENT ON FUNCTION grest.cip129_hex_to_drep_id IS 'Returns DRep Credential ID in CIP-129 format from raw binary hex'; -- noqa: LT01

files/grest/rpc/account/account_info.sql

@@ -3,11 +3,13 @@ RETURNS TABLE (
   stake_address varchar,
   status text,
   delegated_pool varchar,
+  delegated_drep varchar,
   total_balance text,
   utxo text,
   rewards text,
   withdrawals text,
   rewards_available text,
+  deposit text,
   reserves text,
   treasury text
 )
@@ -31,11 +33,13 @@ BEGIN
         'not registered'
       END AS status,
       pool_t.delegated_pool,
+      vote_t.delegated_drep,
       (COALESCE(utxo_t.utxo, 0) + COALESCE(rewards_t.rewards, 0) + COALESCE(reserves_t.reserves, 0) + COALESCE(treasury_t.treasury, 0) - COALESCE(withdrawals_t.withdrawals, 0))::text AS total_balance,
       COALESCE(utxo_t.utxo, 0)::text AS utxo,
       COALESCE(rewards_t.rewards, 0)::text AS rewards,
       COALESCE(withdrawals_t.withdrawals, 0)::text AS withdrawals,
       (COALESCE(rewards_t.rewards, 0) + COALESCE(reserves_t.reserves, 0) + COALESCE(treasury_t.treasury, 0) - COALESCE(withdrawals_t.withdrawals, 0))::text AS rewards_available,
+      COALESCE(status_t.deposit,0)::text AS deposit,
       COALESCE(reserves_t.reserves, 0)::text AS reserves,
       COALESCE(treasury_t.treasury, 0)::text AS treasury
     FROM
@@ -44,28 +48,50 @@ BEGIN
           sa.id,
           sa.view,
           EXISTS (
-            SELECT TRUE FROM stake_registration
-            WHERE
-              stake_registration.addr_id = sa.id
+            SELECT TRUE FROM stake_registration AS sr
+            WHERE sr.addr_id = sa.id
               AND NOT EXISTS (
                 SELECT TRUE
-                FROM stake_deregistration
+                FROM stake_deregistration AS sd
                 WHERE
-                  stake_deregistration.addr_id = stake_registration.addr_id
-                  AND stake_deregistration.tx_id > stake_registration.tx_id
+                  sd.addr_id = sr.addr_id
+                  AND sd.tx_id > sr.tx_id
               )
-          ) AS registered
+          ) AS registered,
+          (
+            SELECT sr.deposit FROM stake_registration AS sr
+            WHERE sr.addr_id = sa.id
+              AND NOT EXISTS (
+                SELECT TRUE
+                FROM stake_deregistration AS sd
+                WHERE
+                  sd.addr_id = sr.addr_id
+                  AND sd.tx_id > sr.tx_id
+              )
+          ) AS deposit
         FROM public.stake_address sa
         WHERE sa.id = ANY(sa_id_list)
       ) AS status_t
+    LEFT JOIN (
+        SELECT
+          dv.addr_id,
+          dh.view AS delegated_drep
+        FROM delegation_vote AS dv
+          INNER JOIN drep_hash AS dh ON dh.id = dv.drep_hash_id
+        WHERE dv.addr_id = ANY(sa_id_list)
+          AND NOT EXISTS (
+            SELECT TRUE
+            FROM delegation_vote AS dv1
+            WHERE dv1.addr_id = dv.addr_id
+              AND dv1.id > dv.id)
+      ) AS vote_t ON vote_t.addr_id = status_t.id
     LEFT JOIN (
         SELECT
           delegation.addr_id,
           pool_hash.view AS delegated_pool
         FROM delegation
           INNER JOIN pool_hash ON pool_hash.id = delegation.pool_hash_id
-        WHERE
-          delegation.addr_id = ANY(sa_id_list)
+        WHERE delegation.addr_id = ANY(sa_id_list)
           AND NOT EXISTS (
             SELECT TRUE
             FROM delegation AS d
@@ -137,7 +163,8 @@ BEGIN
           )
         GROUP BY
           t.addr_id
-      ) AS treasury_t ON treasury_t.addr_id = status_t.id;
+      ) AS treasury_t ON treasury_t.addr_id = status_t.id
+    ;
 END;
 $$;
 

files/grest/rpc/account/account_txs.sql

@@ -28,8 +28,7 @@ BEGIN
     --
     SELECT consumed_by_tx_id AS tx_id
     FROM tx_out
-    WHERE
-      tx_out.consumed_by_tx_id IS NOT NULL
+    WHERE tx_out.consumed_by_tx_id IS NOT NULL
       AND tx_out.stake_address_id = ANY(SELECT id FROM stake_address WHERE view = _stake_address)
       AND tx_out.consumed_by_tx_id >= _tx_id_min
   ) AS tmp;
@@ -43,7 +42,6 @@ BEGIN
     FROM public.tx
     INNER JOIN public.block AS b ON b.id = tx.block_id
     WHERE tx.id = ANY(_tx_id_list)
-      AND b.block_no >= _after_block_height
     ORDER BY b.block_no DESC;
 END;
 $$;

files/grest/rpc/address/address_txs.sql

@@ -11,19 +11,26 @@ DECLARE
   _tx_id_min bigint;
   _tx_id_list bigint[];
 BEGIN
+  SELECT INTO _tx_id_min id
+    FROM tx
+    WHERE block_id >= (SELECT id FROM block WHERE block_no >= _after_block_height ORDER BY id limit 1)
+    ORDER BY id limit 1;
+
   -- all tx_out & tx_in tx ids
   SELECT INTO _tx_id_list ARRAY_AGG(tx_id)
   FROM (
     SELECT tx_id
     FROM tx_out
     WHERE address = ANY (_addresses)
+      AND tx_id >= _tx_id_min
     --
     UNION
     --
     SELECT consumed_by_tx_id
     FROM tx_out
-    WHERE tx_out.address = ANY(_addresses)
-      AND tx_out.consumed_by_tx_id IS NOT NULL
+    WHERE tx_out.consumed_by_tx_id IS NOT NULL
+      AND tx_out.address = ANY(_addresses)
+      AND tx_out.consumed_by_tx_id >= _tx_id_min
   ) AS tmp;
 
   RETURN QUERY
@@ -35,7 +42,6 @@ BEGIN
     FROM public.tx
     INNER JOIN public.block AS b ON b.id = tx.block_id
     WHERE tx.id = ANY(_tx_id_list)
-      AND b.block_no >= _after_block_height
     ORDER BY b.block_no DESC;
 END;
 $$;

files/grest/rpc/address/credential_txs.sql

@@ -50,7 +50,6 @@ BEGIN
     FROM public.tx
     INNER JOIN public.block AS b ON b.id = tx.block_id
     WHERE tx.id = ANY(_tx_id_list)
-      AND b.block_no >= _after_block_height
     ORDER BY b.block_no DESC;
 END;
 $$;