More metrics 2

Makefile

@@ -50,7 +50,7 @@ test_preset: test_deps
 run: deps compile quickrun
 
 quickrun: etc/ejabberd.cfg certs_priv
-	erl -sname mongooseim@localhost -setcookie ejabberd -pa deps/*/ebin apps/*/ebin -config rel/files/app.config -s ejabberd
+	erl -sname mongooseim@localhost -setcookie ejabberd -pa ebin deps/*/ebin apps/*/ebin -config rel/files/app.config -s mongooseim
 
 etc/ejabberd.cfg:
 	@mkdir -p $(@D)

apps/ejabberd/src/ejabberd_c2s.erl

@@ -80,7 +80,6 @@ start_link(SockData, Opts) ->
     ?GEN_FSM:start_link(ejabberd_c2s, [SockData, Opts],
                         fsm_limit_opts(Opts) ++ ?FSMOPTS).
 
-
 socket_type() ->
     xml_stream.
 
@@ -1590,12 +1589,10 @@ change_shaper(StateData, JID) ->
 
 
 -spec send_text(state(), Text :: binary()) -> any().
-send_text(StateData, Text) when StateData#state.xml_socket ->
-    ?DEBUG("Send Text on stream = ~p", [lists:flatten(Text)]),
-    (StateData#state.sockmod):send_xml(StateData#state.socket,
-                                       {xmlstreamraw, Text});
 send_text(StateData, Text) ->
     ?DEBUG("Send XML on stream = ~p", [Text]),
+    Size = size(Text),
+    mongoose_metrics:update([data, xmpp, sent, xml_stanza_size], Size),
     (StateData#state.sockmod):send(StateData#state.socket, Text).
 
 -spec maybe_send_element_safe(state(), El :: jlib:xmlel()) -> any().
@@ -1657,11 +1654,11 @@ send_header(StateData, Server, Version, Lang) ->
             "" -> "";
             _ -> [" xml:lang='", Lang, "'"]
         end,
-    Header = io_lib:format(?STREAM_HEADER,
+    Header = list_to_binary(io_lib:format(?STREAM_HEADER,
                            [StateData#state.streamid,
                             Server,
                             VersionStr,
-                            LangStr]),
+                            LangStr])),
     send_text(StateData, Header).
 
 -spec maybe_send_trailer_safe(State :: state()) -> any().

apps/ejabberd/src/ejabberd_c2s.hrl

@@ -118,7 +118,7 @@
         "id='~s' from='~s'~s~s>"
        ).
 
--define(STREAM_TRAILER, "</stream:stream>").
+-define(STREAM_TRAILER, <<"</stream:stream>">>).
 
 -define(INVALID_NS_ERR, ?SERR_INVALID_NAMESPACE).
 -define(INVALID_XML_ERR, ?SERR_XML_NOT_WELL_FORMED).

apps/ejabberd/src/ejabberd_odbc.erl

@@ -68,6 +68,9 @@
          session_established/2,
          session_established/3]).
 
+%% internal usage
+-export([get_db_info/1]).
+
 -include("ejabberd.hrl").
 
 -record(state, {db_ref,
@@ -168,6 +171,8 @@ keep_alive(PID) ->
     ?GEN_FSM:sync_send_event(PID, {sql_cmd, {sql_query, ?KEEPALIVE_QUERY}, now()},
                              ?KEEPALIVE_TIMEOUT).
 
+get_db_info(Pid) ->
+    ?GEN_FSM:sync_send_all_state_event(Pid, get_db_info).
 %% This function is intended to be used from inside an sql_transaction:
 sql_query_t(Query) ->
     QRes = sql_query_internal(Query),
@@ -379,6 +384,9 @@ session_established(Event, State) ->
 handle_event(_Event, StateName, State) ->
     {next_state, StateName, State}.
 
+handle_sync_event(get_db_info, _, StateName,
+                  #state{db_ref = DbRef, db_type = DbType} = State) ->
+    {reply, {ok, DbType, DbRef}, StateName, State};
 handle_sync_event(_Event, _From, StateName, State) ->
     {reply, {error, badarg}, StateName, State}.
 

apps/ejabberd/src/ejabberd_receiver.erl

@@ -216,9 +216,10 @@ handle_info({Tag, _TCPSocket, Data},
 	    #state{socket = Socket,
 		   c2s_pid = C2SPid,
 		   sock_mod = SockMod} = State)
-  when (Tag == tcp) or (Tag == ssl) or (Tag == ejabberd_xml) ->
+  when (Tag == tcp) or (Tag == ssl) ->
     case SockMod of
 	ejabberd_tls ->
+        mongoose_metrics:update([data, xmpp, received, encrypted_size], size(Data)),
 	    case ejabberd_tls:recv_data(Socket, Data) of
 		{ok, TLSData} ->
 		    {noreply, process_data(TLSData, State),
@@ -227,6 +228,7 @@ handle_info({Tag, _TCPSocket, Data},
 		    {stop, normal, State}
 	    end;
 	ejabberd_zlib ->
+        mongoose_metrics:update([data, xmpp, received, compressed_size], size(Data)),
 	    case ejabberd_zlib:recv_data(Socket, Data) of
 		{ok, ZlibData} ->
 		    {noreply, process_data(ZlibData, State),
@@ -334,8 +336,10 @@ process_data(Data,
                     shaper_state = ShaperState,
                     c2s_pid = C2SPid} = State) ->
     ?DEBUG("Received XML on stream = \"~s\"", [Data]),
+    Size = size(Data),
+    mongoose_metrics:update([data, xmpp, received, xml_stanza_size], Size),
     XMLStreamState1 = xml_stream:parse(XMLStreamState, Data),
-    {NewShaperState, Pause} = shaper:update(ShaperState, size(Data)),
+    {NewShaperState, Pause} = shaper:update(ShaperState, Size),
     if
         C2SPid == undefined ->
             ok;

apps/ejabberd/src/ejabberd_socket.erl

@@ -33,7 +33,6 @@
 	 connect/4,
 	 starttls/2,
 	 starttls/3,
-	 compress/1,
 	 compress/3,
 	 reset_stream/1,
 	 send/2,
@@ -171,16 +170,6 @@ starttls(SocketData, TLSOpts, Data) ->
     send(SocketData, Data),
     SocketData#socket_state{socket = TLSSocket, sockmod = ejabberd_tls}.
 
-
--spec compress(socket_state()) -> socket_state().
-compress(SocketData) ->
-    {ok, ZlibSocket} = ejabberd_zlib:enable_zlib(
-                         SocketData#socket_state.sockmod,
-                         SocketData#socket_state.socket),
-    ejabberd_receiver:compress(SocketData#socket_state.receiver, ZlibSocket),
-    SocketData#socket_state{socket = ZlibSocket, sockmod = ejabberd_zlib}.
-
-
 -spec compress(socket_state(), integer(), _) -> socket_state().
 compress(SocketData, InflateSizeLimit, Data) ->
     {ok, ZlibSocket} = ejabberd_zlib:enable_zlib(

apps/ejabberd/src/ejabberd_sup.erl

@@ -169,6 +169,10 @@ init([]) ->
          brutal_kill,
          worker,
          [mod_muc_iq]},
+    MAM =
+        {mod_mam_sup,
+         {mod_mam_sup, start_link, []},
+         permanent, infinity, supervisor, [mod_mam_sup]},
     ShaperSpecs = shaper_srv:child_specs(),
 
     {ok, {{one_for_one, 10, 1},
@@ -190,4 +194,5 @@ init([]) ->
            IQSupervisor,
            STUNSupervisor,
            Listener,
-           MucIQ]}}.
+           MucIQ,
+           MAM]}}.

apps/ejabberd/src/ejabberd_tls.erl

@@ -149,6 +149,7 @@ send(#tlssock{tcpsock = TCPSocket, tlsport = Port} = TLSSock, Packet) ->
             %?PRINT("OUT: ~p~n", [{TCPSocket, lists:flatten(Packet)}]),
             case port_control(Port, ?GET_ENCRYPTED_OUTPUT, []) of
                 <<0, Out/binary>> ->
+                    mongoose_metrics:update([data, xmpp, sent, encrypted_size], size(Out)),
                     gen_tcp:send(TCPSocket, Out);
                 <<1, Error/binary>> ->
                     {error, binary_to_list(Error)}

apps/ejabberd/src/ejabberd_zlib.erl

@@ -95,6 +95,7 @@ send(#zlibsock{sockmod = SockMod, socket = Socket, zlibport = Port},
      Packet) ->
     case port_control(Port, ?DEFLATE, Packet) of
 	<<0, Out/binary>> ->
+        mongoose_metrics:update([data, xmpp, sent, compressed_size], size(Out)),
 	    SockMod:send(Socket, Out);
 	<<1, Error/binary>> ->
 	    {error, erlang:binary_to_existing_atom(Error, utf8)};

apps/ejabberd/src/gen_mod.erl

@@ -30,6 +30,7 @@
 -export([start/0,
          start_module/3,
          start_backend_module/2,
+         start_backend_module/3,
          stop_module/2,
          stop_module_keep_config/2,
          reload_module/3,
@@ -44,6 +45,7 @@
          loaded_modules_with_opts/1,
          get_hosts/2,
          get_module_proc/2,
+         backend_code/3,
          is_loaded/2]).
 
 -include("ejabberd.hrl").
@@ -107,24 +109,74 @@ start_module(Host, Module, Opts0) ->
 
 -spec start_backend_module(module(), list()) -> no_return().
 start_backend_module(Module, Opts) ->
-    ModuleStr = atom_to_list(Module),
-    BackendModuleStr = ModuleStr ++ "_backend",
+    start_backend_module(Module, Opts, []).
+
+start_backend_module(Module, Opts, TrackedFuncs) ->
     Backend = gen_mod:get_opt(backend, Opts, mnesia),
-    {Mod, Code} = dynamic_compile:from_string(backend_code(ModuleStr, Backend)),
-    code:load_binary(Mod, BackendModuleStr ++ ".erl", Code).
-
--spec backend_code(string(), atom()) -> string().
-backend_code(Module, Backend) when is_atom(Backend) ->
-    BackendModule = Module ++ "_backend",
-    lists:flatten(
-        ["-module(",BackendModule,").
-        -export([backend/0]).
-
-        -spec backend() -> atom().
-        backend() ->",
-            Module,"_",
-            atom_to_list(Backend),
-            ".\n"]).
+    {BackendModuleStr, CodeString} = backend_code(Module, Backend, TrackedFuncs),
+    {Mod, Code} = dynamic_compile:from_string(CodeString),
+    code:load_binary(Mod, BackendModuleStr ++ ".erl", Code),
+    ensure_backend_metrics(Module, TrackedFuncs).
+
+-spec backend_code(string(), atom(), list()) -> string().
+backend_code(Module, Backend, TrackedFuncs) when is_atom(Backend) ->
+    Callbacks = Module:behaviour_info(callbacks),
+    ModuleStr = atom_to_list(Module),
+    BackendModuleName = ModuleStr ++ "_backend",
+    RealBackendModule = ModuleStr++"_"++atom_to_list(Backend),
+    BehaviourExports = [generate_export(F, A) || {F, A} <- Callbacks],
+
+    BehaviourImpl = [generate_fun(Module, RealBackendModule, F, A, TrackedFuncs) || {F, A} <- Callbacks],
+    Code = lists:flatten(
+        ["-module(", BackendModuleName,").\n",
+        "-export([backend/0]).\n",
+        BehaviourExports,
+
+
+        "-spec backend() -> atom().\n",
+        "backend() ->", RealBackendModule,".\n",
+        BehaviourImpl
+        ]),
+    {BackendModuleName, Code}.
+
+generate_export(F, A) ->
+    "-export(["++atom_to_list(F)++"/"++integer_to_list(A)++"]).\n".
+
+generate_fun(BaseModule, RealBackendModule, F, A, TrackedFuncs) ->
+    Args = string:join(["A"++integer_to_list(I) || I <- lists:seq(1, A)], ", "),
+    IsTracked = lists:member(F, TrackedFuncs),
+    [fun_header(F, Args)," ->\n",
+     generate_fun_body(IsTracked, BaseModule, RealBackendModule, F, Args)].
+
+fun_header(F, Args) ->
+    [atom_to_list(F),"(",Args,")"].
+
+-define(METRIC(Module, Op), [backends, Module, Op]).
+
+generate_fun_body(false, _, RealBackendModule, F, Args) ->
+    ["    ",RealBackendModule,":",fun_header(F, Args),".\n"];
+generate_fun_body(true, BaseModule, RealBackendModule, F, Args) ->
+    FS = atom_to_list(F),
+%%     returned is the following
+%%     {Time, Result} = timer:tc(Backend, F, Args),
+%%     mongoose_metrics:update(?METRIC(Backend, F), Time),
+%%     Result.
+    ["    {Time, Result} = timer:tc(",RealBackendModule,", ",FS,", [",Args,"]),\n",
+     "    mongoose_metrics:update(",
+          io_lib:format("~p", [?METRIC(BaseModule, F)]),
+          ", Time),\n",
+     "    Result.\n"].
+
+ensure_backend_metrics(Module, Ops) ->
+    EnsureFun = fun(Op) ->
+        case exometer:info(?METRIC(Module, Op), type) of
+            undefined ->
+                exometer:new(?METRIC(Module, Op), histogram);
+            _ ->
+                ok
+        end
+    end,
+    lists:foreach(EnsureFun, Ops).
 
 -spec is_app_running(_) -> boolean().
 is_app_running(AppName) ->

apps/ejabberd/src/mod_last.erl

@@ -46,7 +46,7 @@
 -include("mod_privacy.hrl").
 -include("mod_last.hrl").
 
--define(BACKEND, (mod_last_backend:backend())).
+-define(BACKEND, mod_last_backend).
 
 %% ------------------------------------------------------------------
 %% Backend callbacks
@@ -82,7 +82,7 @@
 start(Host, Opts) ->
     IQDisc = gen_mod:get_opt(iqdisc, Opts, one_queue),
 
-    gen_mod:start_backend_module(?MODULE, Opts),
+    gen_mod:start_backend_module(?MODULE, Opts, [get_last, set_last_info]),
     ?BACKEND:init(Host, Opts),
 
     gen_iq_handler:add_iq_handler(ejabberd_local, Host,

apps/ejabberd/src/mod_mam_muc_odbc_async_pool_writer.erl

@@ -157,14 +157,14 @@ start_worker(WriterProc, N, Host) ->
      5000,
      worker,
      [mod_mam_muc_odbc_async_writer]},
-    supervisor:start_child(ejabberd_sup, WriterChildSpec).
+    supervisor:start_child(mod_mam_sup, WriterChildSpec).
 
 
 -spec stop_worker(atom()) -> 'ok'
         | {'error','not_found' | 'restarting' | 'running' | 'simple_one_for_one'}.
 stop_worker(Proc) ->
-    supervisor:terminate_child(ejabberd_sup, Proc),
-    supervisor:delete_child(ejabberd_sup, Proc).
+    supervisor:terminate_child(mod_mam_sup, Proc),
+    supervisor:delete_child(mod_mam_sup, Proc).
 
 
 -spec start_link(atom(),_,_) -> 'ignore' | {'error',_} | {'ok',pid()}.
@@ -374,6 +374,8 @@ init([Host, N]) ->
 %%--------------------------------------------------------------------
 -spec handle_call('wait_flushing', _, state())
       -> {'noreply', state()} | {'reply','ok',state()}.
+handle_call(get_connection, _From, State=#state{conn = Conn}) ->
+    {reply, Conn, State};
 handle_call(wait_flushing, _From, State=#state{acc=[]}) ->
     {reply, ok, State};
 handle_call(wait_flushing, From,

apps/ejabberd/src/mod_mam_muc_odbc_async_writer.erl

@@ -219,15 +219,15 @@ is_recent_entries_required(_End, _Now) ->
                                                 | {'ok','undefined' | pid(),_}.
 start_server(Host) ->
     WriterProc = srv_name(Host),
-    supervisor:start_child(ejabberd_sup, writer_child_spec(WriterProc, Host)).
+    supervisor:start_child(mod_mam_sup, writer_child_spec(WriterProc, Host)).
 
 
 -spec stop_server(ejabberd:server()) -> 'ok'
         | {'error','not_found' | 'restarting' | 'running' | 'simple_one_for_one'}.
 stop_server(Host) ->
     Proc = srv_name(Host),
-    supervisor:terminate_child(ejabberd_sup, Proc),
-    supervisor:delete_child(ejabberd_sup, Proc).
+    supervisor:terminate_child(mod_mam_sup, Proc),
+    supervisor:delete_child(mod_mam_sup, Proc).
 
 writer_child_spec(WriterProc, Host) ->
     {WriterProc,
@@ -282,6 +282,8 @@ init([Host]) ->
 %%--------------------------------------------------------------------
 -spec handle_call('wait_flushing',_, state()) -> {'noreply',state()}
                                                | {'reply','ok',state()}.
+handle_call(get_connection, _From, State=#state{conn = Conn}) ->
+    {reply, Conn, State};
 handle_call(wait_flushing, _From, State=#state{acc=[]}) ->
     {reply, ok, State};
 handle_call(wait_flushing, From, State=#state{subscribers=Subs}) ->

apps/ejabberd/src/mod_mam_odbc_async_pool_writer.erl

@@ -175,11 +175,11 @@ start_worker(WriterProc, N, Host) ->
      5000,
      worker,
      [mod_mam_odbc_async_writer]},
-    supervisor:start_child(ejabberd_sup, WriterChildSpec).
+    supervisor:start_child(mod_mam_sup, WriterChildSpec).
 
 stop_worker(Proc) ->
-    supervisor:terminate_child(ejabberd_sup, Proc),
-    supervisor:delete_child(ejabberd_sup, Proc).
+    supervisor:terminate_child(mod_mam_sup, Proc),
+    supervisor:delete_child(mod_mam_sup, Proc).
 
 
 start_link(ProcName, N, Host) ->
@@ -370,6 +370,8 @@ init([Host, N]) ->
 %%                                      {stop, Reason, State}
 %% Description: Handling call messages
 %%--------------------------------------------------------------------
+handle_call(get_connection, _From, State=#state{conn = Conn}) ->
+    {reply, Conn, State};
 handle_call(wait_flushing, _From, State=#state{acc=[]}) ->
     {reply, ok, State};
 handle_call(wait_flushing, From,