print-types.zeek

# Prints out a json specification of all logs generated by system,
# for configuration of zq (https://github.com/brimsec/zq)
#
# * Set environment variable ZEEK_ALLOW_INIT_ERRORS=1 before running Zeek
#   with this script.
#
# * Requires a version of Bro/Zeek with the improvements from:
#   https://github.com/bro/bro/commit/1f450c05102be6dd7ebcc2c5901d5a3a231cd675
#   (Was not included in 2.6 release)
#
# This script is derived from a script by Jon Siwek, available at https://gist.github.com/jsiwek/f843b3321f4227b6ec32d110424ebf70.

module PrintTypes;

type RecColumnType: record {
     name: string;
     typ: string;
};

# We can't create a recursive data type in zeek, so we have a separate
# RecColumnType which mimics ColumnType for nested records. This is a
# awkward and restrictive and we'll need to come up with something
# better. Given that functions _can_ recurse, one approach might be to
# do a recursion through each record, printing out the relevant bits
# during traversal, and then massage the output via another
# script. (Or maybe there's a way by using `any` type that I didn't
# see the first time around...)
type ColumnType: record {
     name: string;
     typ: string &optional;
     rtyp: vector of RecColumnType &optional;
};

function zeek_to_zng_type(zeek: string): string
        {
        if ( /set\[.*\]/ == zeek )
        {
        local inner = zeek_to_zng_type(zeek[4:-1]);
        return "set[" + inner  + "]";
        }

        switch zeek
                {
                case "string":
                        return "bstring";
                case "double":
                        return "float64";
                case "interval":
                        return "duration";
                case "int":
                        return "int64";
                case "count":
                        return "uint64";
                case "addr":
                        return "ip";
                case "subnet":
                        return "net";
                case "enum":
                        return "zenum";
                case "vector":
                        return "array";
                default:
                        return zeek;
                }
        }

function make_nested_column(field: string, field_props: record_field): RecColumnType
	{
	local column: RecColumnType;
	column$name = field;
	if ((/^vector of / in field_props$type_name))
		{
		local split = split_string(field_props$type_name, / of /);
		column$typ = fmt("array[%s]", zeek_to_zng_type(split[1]));
		} else {
		column$typ = zeek_to_zng_type(field_props$type_name);
		}
	return column;
}

function make_column(field: string, field_props: record_field): ColumnType
	{
	local column: ColumnType;
	column$name = field;
	if ((/^vector of / in field_props$type_name))
		{
		local split = split_string(field_props$type_name, / of /);
		column$typ = fmt("array[%s]", zeek_to_zng_type(split[1]));
		} else {
		column$typ = zeek_to_zng_type(field_props$type_name);
		}
	return column;
}

# Adds a nested record to a td, returning false if there is futher nesting below this.
# (See above comment for why we need this hack).
function add_nested_record_to_td(td: vector of RecColumnType, rec_name: string) : bool
	{
	local fields = record_fields(rec_name);
	local field_names = record_type_to_vector(rec_name);

	for ( idx in field_names )
		{
		local field = field_names[idx];
		local field_props = fields[field];

		if ( ! field_props$log )
			next;

		if ((/^record / in field_props$type_name))
			return F;

                td += make_nested_column(field, field_props);
		}
         return T;
}

# Adds record to a td, returning false if there is more than one
# level of nesting below this.
# (See above comment for why we need this hack).
function add_record_to_td(td: vector of ColumnType, rec_name: string) : bool
	{
	local fields = record_fields(rec_name);
	local field_names = record_type_to_vector(rec_name);

	for ( idx in field_names )
		{
		local field = field_names[idx];
		local field_props = fields[field];

		if ( ! field_props$log )
			next;

		if ((/^record / in field_props$type_name))
			{
                        local rec_td: vector of RecColumnType = vector();
			local split = split_string(field_props$type_name, / /);
			if ( add_nested_record_to_td(rec_td, split[1]) == F )
                                return F;
                        td += ColumnType($name=field, $rtyp=rec_td);
			}
			else
			{
			td += make_column(field, field_props);
			}
	}
        return T;
}


event zeek_init() &priority = -100
	{

	local path_to_id_map: table[string] of Log::ID = table();
	local paths: vector of string = vector();
	local stream: Log::Stream;
        local stderr = open("/dev/stderr");


	local tds: table[string] of vector of ColumnType;
	for ( id in Log::active_streams )
		{
		stream = Log::active_streams[id];

		if ( ! stream?$path )
			next;

		local info_id = cat(stream$columns);

		local td: vector of ColumnType = vector();
		td += ColumnType($name="_path", $typ="string");
		if ( add_record_to_td(td, info_id) == F ) {
                        print stderr, fmt("Skipping %s log as it has records nested within records", stream$path);
                        next;
                }
                paths += stream$path;
		td += ColumnType($name="_write_ts", $typ="time");
		local td_name = stream$path+"_log";
		tds[td_name] = td;
		}

	sort(paths, strcmp);
        local descriptors = to_json(tds);
        descriptors = gsub(descriptors, /["]typ["]/, "\"type\"");
        descriptors = gsub(descriptors, /["]rtyp["]/, "\"type\"");

	local output = "";
	output += "{\"descriptors\": ";
        output += descriptors;
	output += ", \"rules\": [";
	local rules: vector of string = vector();
	for ( i in paths )
		{
		local s = "";
		s += "{\"name\":\"_path\", ";
		s += cat("\"value\":\"", paths[i], "\",");
		s += cat("\"descriptor\":\"", paths[i]+"_log", "\"}");
		rules += s;
		}
	output += join_string_vec(rules, ",");
	output += "]}";

	print output;
}