changes to accommodate M&C database

New paradigm:
- M&C DB has authoritative "observation" table
- Librarian DB has clone of this table

Changes
- separate config file for M&C (~/.hera_mc)
- change low-level DB code to handle separate DBs
- change setup scripts to create M&C items
- add PHP and Python bindings of M&C RPCs
- both DBs have "source" and "observation" tables;
    keep code in a common place (db_util.inc)

db_init.php

@@ -1,26 +1,28 @@
 #! /usr/bin/env php
 <?php
 
-// CLI script to create source/site/store records.
+// CLI script to create Librarian and MC records.
 // Usage: db_init.php command args
 //
 // commands:
-// source name
-//      create a source with the given name
+// hl_source name
+//      create a Librarian source with the given name
+// mc_source name
+//      create a MC source with the given name
 // store name capacity
 //      create a store with the given name and capacity (in bytes)
 // test_setup
-//      create some records of each type for testing
+//      create records for testing
 
-require_once("hl_util.inc");
+require_once("hera_util.inc");
 require_once("hl_db.inc");
 
-function create_source($name) {
+function create_source($name, $subsystem) {
     $source = new StdClass;
     $source->name = $name;
     $source->authenticator = random_string();
     $source->create_time = time();
-    echo "creating source $name; authenticator: $source->authenticator\n";
+    echo "creating $subsystem source $name; authenticator: $source->authenticator\n";
     return source_insert($source);
 }
 
@@ -33,9 +35,19 @@ function create_store($name, $capacity) {
     return store_insert($store);
 }
 
-function test_setup() {
+function mc_setup() {
+    init_db(MC_DB_NAME);
     foreach (array('RTP', 'raw data') as $u) {
-        if (!create_source($u)) {
+        if (!create_source($u, 'M&C')) {
+            echo db_error()."\n";
+        }
+    }
+}
+
+function hl_setup() {
+    init_db(LIBRARIAN_DB_NAME);
+    foreach (array('RTP', 'raw data') as $u) {
+        if (!create_source($u, 'Librarian')) {
             echo db_error()."\n";
         }
     }
@@ -46,12 +58,24 @@ function test_setup() {
     }
 }
 
-init_db();
 
 switch ($argv[1]) {
-case 'source': create_source($argv[2]); break;
-case 'store': create_store($argv[2], $argv[3], $argv[4]); break;
-case 'test_setup': test_setup(); break;
+case 'hl_source':
+    init_db(LIBRARIAN_DB_NAME);
+    create_source($argv[2]);
+    break;
+case 'mc_source':
+    init_db(MC_DB_NAME);
+    create_source($argv[2]);
+    break;
+case 'store':
+    init_db(LIBRARIAN_DB_NAME);
+    create_store($argv[2], $argv[3], $argv[4]);
+    break;
+case 'test_setup':
+    mc_setup();
+    hl_setup();
+    break;
 default: die("no such command\n");
 }
 

db_util.inc

@@ -0,0 +1,95 @@
+<?php
+
+// MySQL code for MC and Librarian DBs
+
+$link = null;
+$db_error = '';
+define('LIBRARIAN_DB_NAME', 'hera_lib');
+define('MC_DB_NAME', 'hera_mc');
+
+function init_db($db_name) {
+    global $link;
+    $link = @mysqli_connect('', '', '', $db_name);
+    return $link;
+}
+
+function insert_id() {
+    global $link;
+    return mysqli_insert_id($link);
+}
+
+function db_error() {
+    global $link;
+    return mysqli_error($link);
+}
+
+function enum($table, $clause=null) {
+    global $link;
+    if (!$clause) $clause = 'TRUE';
+    $query = "select * from $table where $clause";
+    $r = mysqli_query($link, $query);
+    $items = array();
+    while ($f = mysqli_fetch_object($r)) {
+        $items[] = $f;
+    }
+    mysqli_free_result($r);
+    return $items;
+}
+
+function lookup_id($table, $id) {
+    global $link;
+    $query = "select * from $table where id=$id";
+    $r = mysqli_query($link, $query);
+    $source = mysqli_fetch_object($r);
+    mysqli_free_result($r);
+    return $source;
+}
+
+//// SOURCE ////
+
+function source_insert($source) {
+    global $link;
+    $auth = $link->escape_string($source->authenticator);
+    $query = "insert into source (name, authenticator, create_time) values ('$source->name', '$auth', $source->create_time)";
+    return mysqli_query($link, $query);
+}
+
+function source_lookup_auth($auth) {
+    global $link;
+    $auth = $link->escape_string($auth);
+    $query = "select * from source where authenticator='$auth'";
+    $r = mysqli_query($link, $query);
+    $source = mysqli_fetch_object($r);
+    mysqli_free_result($r);
+    return $source;
+}
+
+function source_lookup_id($id) {
+    return lookup_id("source", $id);
+}
+
+function source_enum() {
+    return enum('source');
+}
+
+//// OBSERVATION ////
+
+function observation_insert_hl($obs) {
+    global $link;
+    $polarization = $link->escape_string($obs->polarization);
+    $query = "insert into observation (id, source_id, julian_date, polarization, length_days) values ($obs->id, $obs->source_id, $obs->julian_date, '$polarization', $obs->length_days)";
+    return mysqli_query($link, $query);
+}
+
+function observation_insert_mc($obs) {
+    global $link;
+    $polarization = $link->escape_string($obs->polarization);
+    $query = "insert into observation (source_id, julian_date, polarization, length_days) values ($obs->source_id, $obs->julian_date, '$polarization', $obs->length_days)";
+    return mysqli_query($link, $query);
+}
+
+function observation_lookup_id($id) {
+    return lookup_id("observation", $id);
+}
+
+?>

hera_librarian.py

@@ -1,42 +1,15 @@
-import os
-import urllib
-import json
-
-# parse config file, return as a dictionary
-#
-def get_config():
-    path = os.path.expanduser('~/.hera_librarian')
-    f = open(path, 'r')
-    config = {}
-    for line in f:
-        wds = line.split()
-        config[wds[0]] = wds[1]
-    f.close()
-    return config
-
-# do a POST operation,
-# passing a JSON version of the request and expecting a JSON reply;
-# return the decoded version of the latter.
-#
-def do_http_post(req, config):
-    req_json = json.dumps(req)
-    params = urllib.urlencode({'request': req_json})
-    url = config['server']+'/hl_rpc_handler.php'
-    f = urllib.urlopen(url , params);
-    reply_json = f.read()
-    reply = json.loads(reply_json)
-    return reply
+import hera_rpc
 
 # RPC to create a file
 #
 def create_file(name, size, md5, store_name):
-    config = get_config()
+    config = get_config('.hera_librarian')
     req = {'operation': 'create_file',
         'authenticator': config['authenticator'],
         'name': name,
         'size': size,
         'md5': md5,
         'store_name': store_name}
-    return do_http_post(req, config)
+    return do_http_post(req, config['server'])
 
 #create_file('filename2', 2e9, 'ajfjfkdjffjf', 'UCB RAID')

hera_mc.py

@@ -0,0 +1,15 @@
+import hera_rpc
+
+# Python binding of M&C RPCs
+
+# create an observation
+#
+def create_observation(julian_date, polarization, length):
+    config = get_config('.hera_mc')
+    req = {'operation': 'create_observation',
+        'authenticator': config['authenticator'],
+        'name': name,
+        'size': size,
+        'md5': md5,
+        'store_name': store_name}
+    return do_http_post(req, config['server'])

hera_rpc.py

@@ -0,0 +1,31 @@
+# utility functions for Python RPC bindings
+
+import os
+import urllib
+import json
+
+# parse config file, return as a dictionary
+#
+def get_config(file):
+    path = os.path.expanduser('~/'+file)
+    f = open(path, 'r')
+    config = {}
+    for line in f:
+        wds = line.split()
+        config[wds[0]] = wds[1]
+    f.close()
+    return config
+
+# do a POST operation,
+# passing a JSON version of the request and expecting a JSON reply;
+# return the decoded version of the latter.
+#
+def do_http_post(req, server):
+    req_json = json.dumps(req)
+    params = urllib.urlencode({'request': req_json})
+    url = server+'/hl_rpc_handler.php'
+    f = urllib.urlopen(url , params);
+    reply_json = f.read()
+    reply = json.loads(reply_json)
+    return reply
+

hl_util.inc → hera_util.inc

@@ -4,11 +4,11 @@ error_reporting(E_ALL);
 ini_set('display_errors', true);
 ini_set('display_startup_errors', true);
 
-// parse Librarian config file, return as object
+// parse config file, return as object
 //
-function get_config() {
+function get_config($file) {
     $home = getenv('HOME');
-    $config_path = "$home/.hera_librarian";
+    $config_path = "$home/$file";
     $lines = file($config_path);
     if (!$lines) {
         die("can't read $config_path\n");

hl.php

@@ -18,14 +18,14 @@ function show_source_select() {
     ';
 }
 
-function show_site_select() {
+function show_store_select() {
     echo '<div class="form-group">
-        <label for="site_id">Site:</label>
-        <select name=site_id>
+        <label for="store_id">Store:</label>
+        <select name=store_id>
         <option value=0> All
     ';
-    $sites = site_enum();
-    foreach ($sites as $s) {
+    $stores = store_enum();
+    foreach ($stores as $s) {
         echo "<option value=$s->id> $s->name\n";
     }
     echo '</div>
@@ -48,39 +48,35 @@ function file_search_form() {
 function file_search_action() {
     page_head("Files");
     table_start();
-    table_header(array("Name", "Created", "Source", "Size", "Site", "Store"));
+    table_header(array("Name", "Created", "Source", "Size", "Store"));
     $clause = '';
     $source_id = get_int('source_id');
     if ($source_id) {
         $clause = "file.source_id = $source_id";
     }
-    $fis = file_instance_enum2($clause);
-    foreach ($fis as $fi) {
-        $source = source_lookup_id($fi->f_source_id);
-        $store = store_lookup_id($fi->store_id);
-        $site = site_lookup_id($store->site_id);
+    $files = file_enum($clause);
+    foreach ($files as $file) {
+        $source = source_lookup_id($file->source_id);
+        $store = store_lookup_id($file->store_id);
         table_row(array(
-            $fi->f_name,
-            time_str($fi->create_time),
+            $file->name,
+            time_str($file->create_time),
             $source->name,
-            size_str($fi->f_size),
-            $site->name,
+            size_str($file->size),
             $store->name
         ));
     }
     table_end();
     page_tail();
 }
 
-function show_storage() {
+function show_stores() {
     page_head("Storage");
     table_start();
-    table_header(array("Site", "Name", "Capacity", "Used", "% used"));
+    table_header(array("Name", "Capacity", "Used", "% used"));
     $stores = store_enum();
     foreach ($stores as $store) {
-        $site = site_lookup_id($store->site_id);
         table_row(array(
-            $site->name,
             $store->name,
             size_str($store->capacity),
             size_str($store->used),
@@ -91,16 +87,16 @@ function show_storage() {
     page_tail();
 }
 
-if (!init_db()) {
+if (!init_db(LIBRARIAN_DB_NAME)) {
     error_page("can't open DB");
 }
 
 $action = get_str("action", true);
 switch ($action) {
 case 'search':
     file_search_action(); break;
-case 'storage':
-    show_storage(); break;
+case 'stores':
+    show_stores(); break;
 default:
     file_search_form(); break;
 }