Sendfile handler: can use {host} variable when specifying document ro…

…ot directory. Image filter can optionally serialize calls to ImageMagick (with mutex) to minimize memory usage. Fixed rare bug in SSI and templates code. More robust time parsing using timegm().

sample_config/modules/subrequests.c

@@ -39,28 +39,6 @@ static nxweb_result subreq_on_request(nxweb_http_server_connection* conn, nxweb_
 
 NXWEB_DEFINE_HANDLER(subreq, .on_request=subreq_on_request, .flags=NXWEB_HANDLE_ANY);
 
-int nxt_parse(nxt_context* ctx, const char* uri, char* buf, int buf_len);
-
-static int tmpl_load(nxt_context* ctx, const char* uri, nxt_file* dst_file, nxt_block* dst_block) { // function to make subrequests
-  if (dst_block) {
-    nxweb_log_info("including file %s", uri);
-    nxt_block_append_value(ctx, dst_block, "{% This is included file %}", sizeof("{% This is included file %}")-1, 0);
-  }
-  else {
-    nxweb_log_info("loading template from %s", uri);
-    if (!strcmp(uri, "base")) {
-      const char* tmpl_src=" {%block _top_%}{%raw%}{{{{%%%%}}}}{%endraw%}{% include aaa %}AAA-YYY{%block header%}Header{%endblock%} bye...{% endblock %}{% block title %}New Title{% endblock %}";
-      char* tmpl=nxb_copy_obj(ctx->nxb, tmpl_src, strlen(tmpl_src)+1);
-      nxt_parse_file(dst_file, (char*)tmpl, strlen(tmpl));
-    }
-    else if (!strcmp(uri, "ttt")) {
-      const char* tmpl_src=" {% extends 'base'%} {%block header%}Bbbb {%block title%}{%endblock%} {% parent %}{%endblock%}";
-      char* tmpl=nxb_copy_obj(ctx->nxb, tmpl_src, strlen(tmpl_src)+1);
-      nxt_parse_file(dst_file, (char*)tmpl, strlen(tmpl));
-    }
-  }
-}
-
 static nxweb_result curtime_on_request(nxweb_http_server_connection* conn, nxweb_http_request* req, nxweb_http_response* resp) {
   nxweb_set_response_content_type(resp, "text/html");
 

src/include/nxweb/http_server.h

@@ -55,7 +55,8 @@ typedef enum nxweb_handler_flags {
   NXWEB_HANDLE_OTHER=0x40,
   NXWEB_HANDLE_ANY=0x70,
   NXWEB_ACCEPT_CONTENT=0x80, // handler accepts request body
-  _NXWEB_HANDLE_MASK=0x70
+  _NXWEB_HANDLE_MASK=0x70,
+  _NXWEB_HOST_DEPENDENT_DIR=0x1000 // sendfile handler to use host name to build root directory
 } nxweb_handler_flags;
 
 struct nxweb_http_server_connection;

src/lib/filters/image_filter.c

@@ -93,6 +93,7 @@ typedef struct nxweb_filter_image {
   const char* cache_dir;
   nxweb_image_filter_cmd* allowed_cmds;
   const char* sign_key;
+  _Bool use_lock:1;
 } nxweb_filter_image;
 
 /**
@@ -117,9 +118,12 @@ static nxweb_image_filter_cmd default_allowed_cmds[]={
   {.cmd=0}
 };
 
+static pthread_mutex_t image_filter_processing_lock;
 
 static int image_filter_init() {
   MagickWandGenesis();
+  nxweb_log_error("MagickResourceLimits: mem=%lu, map=%lu, area=%lu", (long)MagickGetResourceLimit(MemoryResource), (long)MagickGetResourceLimit(MemoryResource), (long)MagickGetResourceLimit(AreaResource));
+  pthread_mutex_init(&image_filter_processing_lock, 0);
   return 0;
 }
 
@@ -682,6 +686,11 @@ static nxweb_result img_translate_cache_key(nxweb_filter* filter, nxweb_http_ser
   return NXWEB_OK;
 }
 
+static inline void _cleanup_magick_wand(MagickWand* image, _Bool use_lock) {
+  DestroyMagickWand(image);
+  if (use_lock) pthread_mutex_unlock(&image_filter_processing_lock);
+}
+
 static nxweb_result img_do_filter(nxweb_filter* filter, nxweb_http_server_connection* conn, nxweb_http_request* req, nxweb_http_response* resp, nxweb_filter_data* fdata) {
   img_filter_data* ifdata=(img_filter_data*)fdata;
   if (resp->status_code && resp->status_code!=200) return NXWEB_OK;
@@ -690,10 +699,11 @@ static nxweb_result img_do_filter(nxweb_filter* filter, nxweb_http_server_connec
   assert(resp->sendfile_path);
   assert(resp->content_length>0 && resp->sendfile_offset==0 && resp->sendfile_end==resp->sendfile_info.st_size &&  resp->sendfile_end==resp->content_length);
 
-  assert(((nxweb_filter_image*)filter)->cache_dir);
+  nxweb_filter_image* ifilter=(nxweb_filter_image*)filter;
+  assert(ifilter->cache_dir);
   nxb_buffer* nxb=req->nxb;
   nxb_start_stream(nxb);
-  nxb_append_str(nxb, ((nxweb_filter_image*)filter)->cache_dir);
+  nxb_append_str(nxb, ifilter->cache_dir);
   const char* cache_key=ifdata->fdata.cache_key;
   if (cache_key[0]=='.' && cache_key[1]=='.' && cache_key[2]=='/') cache_key+=2; // avoid going up dir tree
   if (*cache_key!='/') nxb_append_char(nxb, '/');
@@ -718,7 +728,7 @@ static nxweb_result img_do_filter(nxweb_filter* filter, nxweb_http_server_connec
   if (stat(fpath, &finfo)==-1 || finfo.st_mtime!=resp->sendfile_info.st_mtime) {
     if (ifdata->cmd.cmd) {
       // check if cmd is allowed
-      const nxweb_image_filter_cmd* ac=((nxweb_filter_image*)filter)->allowed_cmds;
+      const nxweb_image_filter_cmd* ac=ifilter->allowed_cmds;
       _Bool allowed=0;
       while (ac->cmd) {
         if (ifdata->cmd.cmd==ac->cmd && ifdata->cmd.width==ac->width && ifdata->cmd.height==ac->height
@@ -732,7 +742,7 @@ static nxweb_result img_do_filter(nxweb_filter* filter, nxweb_http_server_connec
       }
       if (!allowed && ifdata->cmd.query_string) {
         char signature[41];
-        sha1sign(ifdata->cmd.uri_path, strlen(ifdata->cmd.uri_path), ((nxweb_filter_image*)filter)->sign_key, signature);
+        sha1sign(ifdata->cmd.uri_path, strlen(ifdata->cmd.uri_path), ifilter->sign_key, signature);
         if (!strcmp(ifdata->cmd.query_string, signature)) allowed=1;
         else nxweb_log_warning("img cmd not allowed: path=%s sha1sign=%s query_string=%s", ifdata->cmd.uri_path, signature, ifdata->cmd.query_string);
       }
@@ -747,28 +757,41 @@ static nxweb_result img_do_filter(nxweb_filter* filter, nxweb_http_server_connec
       return NXWEB_ERROR;
     }
 
+    _Bool use_lock=ifilter->use_lock; // filter config option
+    if (use_lock) // serialize calls to ImageMagick to minimize memory footprint; not good for performance
+      pthread_mutex_lock(&image_filter_processing_lock);
+
     MagickWand* image=NewMagickWand();
     if (!MagickReadImage(image, resp->sendfile_path)) {
-      DestroyMagickWand(image);
+      _cleanup_magick_wand(image, use_lock);
       nxweb_log_error("MagickReadImage(%s) failed", resp->sendfile_path);
       nxweb_send_http_error(resp, 500, "Internal Server Error");
       return NXWEB_ERROR;
     }
 
-    if (process_cmd(resp->sendfile_path, conn->handler->dir? strlen(conn->handler->dir) : 0, image, &ifdata->cmd)) {
+    int doc_root_len;
+    if (conn->handler->flags & _NXWEB_HOST_DEPENDENT_DIR) {
+      const char* port=strchr(req->host, ':');
+      int host_len=port? port - req->host : strlen(req->host);
+      doc_root_len=strlen(conn->handler->dir)-(sizeof("{host}")-1)+host_len;
+    }
+    else {
+      doc_root_len=conn->handler->dir? strlen(conn->handler->dir) : 0;
+    }
+    if (process_cmd(resp->sendfile_path, doc_root_len, image, &ifdata->cmd)) {
       nxweb_log_info("writing image file %s", fpath);
       MagickStripImage(image);
       if (!MagickWriteImage(image, fpath)) {
-        DestroyMagickWand(image);
+        _cleanup_magick_wand(image, use_lock);
         nxweb_log_error("MagickWriteImage(%s) failed", fpath);
         nxweb_send_http_error(resp, 500, "Internal Server Error");
         return NXWEB_ERROR;
       }
-      DestroyMagickWand(image);
+      _cleanup_magick_wand(image, use_lock);
     }
     else {
       // processing changed nothing => just copy the original
-      DestroyMagickWand(image);
+      _cleanup_magick_wand(image, use_lock);
       nxweb_log_info("copying image file %s", fpath);
       if (copy_file(resp->sendfile_path, fpath)) {
         nxweb_log_error("error %d copying image file %s", errno, fpath);
@@ -816,6 +839,7 @@ static nxweb_filter* img_config(nxweb_filter* base, const nx_json* json) {
   f->cache_dir=nx_json_get(json, "cache_dir")->text_value;
   const char* sign_key=nx_json_get(json, "sign_key")->text_value;
   if (sign_key) f->sign_key=sign_key;
+  f->use_lock=!!nx_json_get(json, "use_lock")->int_value;
   const nx_json* allowed_cmds_json=nx_json_get(json, "allowed_cmds");
   if (allowed_cmds_json->type!=NX_JSON_NULL) {
     nxweb_image_filter_cmd* allowed_cmds=calloc(allowed_cmds_json->length+1, sizeof(nxweb_image_filter_cmd));

src/lib/filters/templates_filter.c

@@ -125,7 +125,6 @@ static nxe_ssize_t tf_buffer_data_in_write(nxe_ostream* os, nxe_istream* is, int
       if (tfb->file) {
         if (nxt_parse_file(tfb->file, ptr, size)==-1) {
           // handle error
-          ctx->files_pending--;
           // it's been logged; ignore it
         }
       }
@@ -176,7 +175,7 @@ static void tf_on_subrequest_ready(nxweb_http_server_connection* subconn, nxe_da
       if (!resp->last_modified) tfdata->last_modified=0;
       else if (resp->last_modified > tfdata->last_modified) tfdata->last_modified=resp->last_modified;
     }
-    if (resp->content_length==0) {
+    if (resp->content_length==0) { // empty response => nothing to parse/include
       tfdata->ctx->files_pending--;
       tf_check_complete(tfdata);
     }
@@ -213,9 +212,11 @@ static int tf_load(nxt_context* ctx, const char* uri, nxt_file* dst_file, nxt_bl
     tf_buffer_init(tfb, ctx->nxb, tfdata, dst_file, 0);
   }
   nxweb_http_server_connection* subconn=nxweb_http_server_subrequest_start(tfdata->conn, tf_on_subrequest_ready, (nxe_data)(void*)tfb, 0, uri);
+  if (!subconn) return NXWEB_ERROR;
   nxweb_http_request* subreq=&subconn->hsp.req;
   nxweb_set_request_data(subreq, (nxe_data)0, (nxe_data)(void*)tfb, tf_subreq_finalize);
   if (dst_file) subreq->templates_no_parse=1;
+  return NXWEB_OK;
 }
 
 static nxweb_filter_data* tf_init(nxweb_filter* filter, nxweb_http_server_connection* conn, nxweb_http_request* req, nxweb_http_response* resp) {

src/lib/http_server.c

@@ -337,6 +337,8 @@ void _nxweb_register_handler(nxweb_handler* handler, nxweb_handler* base) {
     }
   }
   handler->vhost_len=handler->vhost? strlen(handler->vhost) : 0;
+  if (handler->dir && strstr(handler->dir, "{host}")) handler->flags|=_NXWEB_HOST_DEPENDENT_DIR;
+  else handler->flags&=~_NXWEB_HOST_DEPENDENT_DIR;
   if (base) {
     if (!handler->on_generate_cache_key) handler->on_generate_cache_key=base->on_generate_cache_key;
     if (!handler->on_select) handler->on_select=base->on_select;

src/lib/http_subrequest.c

@@ -109,6 +109,12 @@ void nxweb_composite_stream_append_subrequest(nxweb_composite_stream* cs, const
   nxweb_composite_stream_node* csn=nxweb_composite_stream_append_node(cs);
 
   csn->subconn=nxweb_http_server_subrequest_start(cs->conn, nxweb_composite_stream_subrequest_on_response_ready, (nxe_data)(void*)cs, host, url);
+  if (!csn->subconn) {
+    nxweb_log_error("nxweb_http_server_subrequest_start failed: %s %s %d", host, url, (int)cs->conn->connection_closing);
+    // append bytes instead
+    nxd_obuffer_init(&csn->buffer.ob, "<!--[subrequest failed]-->", sizeof("<!--[subrequest failed]-->")-1);
+    nxe_connect_streams(cs->conn->tdata->loop, &csn->buffer.ob.data_out, &csn->snode.data_in);
+  }
 }
 
 void nxweb_composite_stream_close(nxweb_composite_stream* cs) { // call this right after appending last node

src/lib/http_utils.c

@@ -159,7 +159,7 @@ time_t nxweb_parse_http_time(const char* str) { // must be GMT
   if (*p++!='M') return 0;
   if (*p++!='T') return 0;
   tm.tm_isdst=-1; // auto-detect
-  time_t t=mktime(&tm) - timezone;
+  time_t t=timegm(&tm); // mktime(&tm) - timezone;
   if (t==-1) return 0;
   return t;
 }

src/lib/modules/sendfile.c

@@ -29,14 +29,31 @@ static nxweb_result sendfile_generate_cache_key(nxweb_http_server_connection* co
   if (!req->get_method || req->content_length) return NXWEB_NEXT; // do not respond to POST requests, etc.
 
   nxweb_handler* handler=conn->handler;
-  const char* document_root=handler->dir;
-  assert(document_root);
+  assert(handler->dir);
   assert(handler->index_file);
+  int rlen;
   nxb_buffer* nxb=req->nxb;
-  nxb_start_stream(nxb);
-
-  int rlen=strlen(document_root);
-  nxb_append(nxb, document_root, rlen);
+  if (handler->flags & _NXWEB_HOST_DEPENDENT_DIR) {
+    // validate host name
+    const char* host=req->host;
+    if (*host=='.' || strchr(host, '/')) { // invalid hostname, prevent possible attack
+      return NXWEB_NEXT;
+    }
+    const char* port=strchr(host, ':');
+    int host_len=port? port - host : strlen(host);
+    const char* host_placeholder=strstr(handler->dir, "{host}");
+    assert(host_placeholder); // _NXWEB_HOST_DEPENDENT_DIR flag is only set when {host} is present
+    nxb_start_stream(nxb);
+    nxb_append(nxb, handler->dir, host_placeholder - handler->dir);
+    nxb_append(nxb, host, host_len);
+    nxb_append_str(nxb, host_placeholder+sizeof("{host}")-1);
+    rlen=strlen(handler->dir)-(sizeof("{host}")-1)+host_len;
+  }
+  else {
+    rlen=strlen(handler->dir);
+    nxb_start_stream(nxb);
+    nxb_append(nxb, handler->dir, rlen);
+  }
   const char* q=strchr(req->path_info, '?');
   int plen=q? q-req->path_info : strlen(req->path_info);
   nxb_append(nxb, req->path_info, plen);
@@ -77,8 +94,21 @@ static nxweb_result sendfile_on_select(nxweb_http_server_connection* conn, nxweb
   }
 
   if (S_ISDIR(finfo->st_mode)) {
-    const char* path_info=fpath+strlen(conn->handler->dir);
-    nxweb_send_redirect2(resp, 302, path_info, "/", conn->secure);
+    // this is directory but no trailing slash in uri => append '/' to the end of path and redirect
+    const char* q=strchr(req->uri, '?');
+    if (q) { // have query string
+      nxb_buffer* nxb=resp->nxb;
+      nxb_start_stream(nxb);
+      nxb_append(nxb, req->uri, q - req->uri);
+      nxb_append_char(nxb, '/'); // insert '/' at the end of path before query string
+      nxb_append_str(nxb, q);
+      nxb_append_char(nxb, '\0');
+      const char* new_uri=nxb_finish_stream(nxb, 0);
+      nxweb_send_redirect(resp, 302, new_uri, conn->secure);
+    }
+    else {
+      nxweb_send_redirect2(resp, 302, req->uri, "/", conn->secure);
+    }
     nxweb_start_sending_response(conn, resp);
     return NXWEB_OK;
   }