Merge branch 'master' into parallel-http-join

netios · Nov 30, 2017 · 38fc621 · 38fc621
2 parents 52ea223 + 5244fd5
commit 38fc621
Show file tree

Hide file tree

Showing 32 changed files with 592 additions and 124 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -36,6 +36,8 @@ SET(LIBRARY_VERSION "${LIBRARY_VERSION_MAJOR}.${LIBRARY_VERSION_MINOR}.${LIBRARY
 SET(LIBRARY_SOVERSION "${LIBRARY_VERSION_MAJOR}.${LIBRARY_VERSION_MINOR}")
 
 INCLUDE(GNUInstallDirs)
+INCLUDE(CheckCSourceCompiles)
+INCLUDE(CMakePushCheckState)
 
 CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/version.h.in ${CMAKE_CURRENT_SOURCE_DIR}/include/h2o/version.h)
 CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/libh2o.pc.in ${CMAKE_CURRENT_BINARY_DIR}/libh2o.pc @ONLY)
@@ -49,7 +51,6 @@ FIND_PACKAGE(Threads REQUIRED)
 FIND_PACKAGE(OpenSSL)
 FIND_PACKAGE(ZLIB REQUIRED)
 
-INCLUDE(CheckCSourceCompiles)
 CHECK_C_SOURCE_COMPILES("
 #include <stdint.h>
 int main(void) {
@@ -58,6 +59,32 @@ __sync_add_and_fetch(&a, 1);
 return 0;
 }" ARCH_SUPPORTS_64BIT_ATOMICS)
 
+
+
+# Find out if libc provices backtrace()
+CMAKE_PUSH_CHECK_STATE()
+FIND_LIBRARY(LIBC_BACKTRACE_LIB "execinfo")
+IF (LIBC_BACKTRACE_LIB)
+    SET(CMAKE_REQUIRED_LIBRARIES ${LIBC_BACKTRACE_LIB})
+ENDIF()
+CHECK_C_SOURCE_COMPILES("
+#include <execinfo.h>
+int main(void) {
+void *p[10];
+int ret = backtrace(p, 10);
+backtrace_symbols_fd(p, ret, 2);
+return 0;
+}" LIBC_HAS_BACKTRACE)
+
+CMAKE_POP_CHECK_STATE()
+
+IF (LIBC_HAS_BACKTRACE)
+    ADD_DEFINITIONS("-DLIBC_HAS_BACKTRACE")
+    IF (LIBC_BACKTRACE_LIB)
+        LIST(APPEND EXTRA_LIBS ${LIBC_BACKTRACE_LIB})
+    ENDIF ()
+ENDIF ()
+
 SET(WITH_BUNDLED_SSL_DEFAULT "ON")
 IF ((NOT UNIX) OR CYGWIN)
     SET(WITH_BUNDLED_SSL_DEFAULT "OFF")

diff --git a/doc/configure/cgi.html b/doc/configure/cgi.html
@@ -79,7 +79,8 @@ <h2>
 </div>
 
 
-The gateway also provides options to for tuning the behavior.  A full list of options can be obtained by running the gateway directly with <code>--help</code> option.
+The gateway also provides options to for tuning the behavior.
+A full list of options can be obtained by running the gateway directly with <code>--help</code> option.
 
 <div class="example">
 <div class="caption">Example. Output of <code>share/h2o/fastcgi-cgi --help</code></div>
@@ -95,6 +96,7 @@ <h2>
                      connections.
   --max-workers=nnn  maximum number of CGI processes (default: unlimited)
   --pass-authz       if set, preserves HTTP_AUTHORIZATION parameter
+  --verbose          verbose mode
 </code></pre>
 </div>
 

diff --git a/doc/configure/proxy_directives.html b/doc/configure/proxy_directives.html
@@ -227,6 +227,26 @@ <h3><a href="configure/proxy_directives.html#proxy.emit-via-header"><code>"proxy
 </dl>
 
 
+<div id="proxy.emit-missing-date-header" class="directive-head">
+<div class="directive-since">since v2.3</div>
+<h3><a href="configure/proxy_directives.html#proxy.emit-missing-date-header"><code>"proxy.emit-missing-date-header"</code></a></h3>
+</div>
+
+<dl class="directive-desc">
+<dt>Description:</dt>
+<dd>
+<p>
+A boolean flag (<code>ON</code> or <code>OFF</code>) indicating if H2O should add a <code>date</date> header to the response, if that header is missing from the upstream response.
+</p>
+
+</dd>
+<dt><a href="configure/syntax_and_structure.html#config_levels">Level</a>:</dt>
+<dd>global</dd>
+<dt>Default:</dt>
+<dd><code><pre>proxy.emit-missing-date-header: ON</pre></code>
+</dl>
+
+
 <div id="proxy.header.add" class="directive-head">
 <div class="directive-since">since v2.2</div>
 <h3><a href="configure/proxy_directives.html#proxy.header.add"><code>"proxy.header.add"</code></a></h3>

diff --git a/doc/h2o.conf.5 b/doc/h2o.conf.5
@@ -2617,6 +2617,16 @@ proxy.emit-x-forwarded-headers
 .RE
 
 
+.SS proxy.emit-missing-date-header
+(since v2.3)
+A boolean flag (ON or OFF) indicating if H2O should add a date header to the response, if that header is missing from the upstream response.
+
+.PP
+
+.RE
+.RE
+
+
 .SS proxy.header.add
 (since v2.2)
 Modifies the request headers sent to the application server.
@@ -3113,7 +3123,8 @@ file.custom-handler:
 .PP
 
 
-The gateway also provides options to for tuning the behavior.  A full list of options can be obtained by running the gateway directly with --help option.
+The gateway also provides options to for tuning the behavior.
+A full list of options can be obtained by running the gateway directly with --help option.
 
 .PP
 .BR Example:\ 
@@ -3133,6 +3144,7 @@ Options:
                      connections.
   --max-workers=nnn  maximum number of CGI processes (default: unlimited)
   --pass-authz       if set, preserves HTTP_AUTHORIZATION parameter
+  --verbose          verbose mode
 
 .RE
 .fi

diff --git a/doc/search/searchindex.js b/doc/search/searchindex.js
diff --git a/include/h2o.h b/include/h2o.h
@@ -420,6 +420,10 @@ struct st_h2o_globalconf_t {
          * a boolean flag if set to true, instructs the proxy to emit a via header
          */
         unsigned emit_via_header : 1;
+        /**
+         * a boolean flag if set to true, instructs the proxy to emit a date header, if it's missing from the upstream response
+         */
+        unsigned emit_missing_date_header : 1;
         /**
          * global socketpool
          */
@@ -1133,12 +1137,14 @@ typedef struct st_h2o_accept_ctx_t {
 
 typedef struct st_h2o_doublebuffer_t {
     h2o_buffer_t *buf;
-    size_t bytes_inflight;
+    unsigned char inflight : 1;
+    size_t _bytes_inflight;
 } h2o_doublebuffer_t;
 
 static void h2o_doublebuffer_init(h2o_doublebuffer_t *db, h2o_buffer_prototype_t *prototype);
 static void h2o_doublebuffer_dispose(h2o_doublebuffer_t *db);
 static h2o_iovec_t h2o_doublebuffer_prepare(h2o_doublebuffer_t *db, h2o_buffer_t **receiving, size_t max_bytes);
+static void h2o_doublebuffer_prepare_empty(h2o_doublebuffer_t *db);
 static void h2o_doublebuffer_consume(h2o_doublebuffer_t *db);
 
 /* token */
@@ -1157,6 +1163,7 @@ int h2o_iovec_is_token(const h2o_iovec_t *buf);
 
 /* headers */
 
+static int h2o_header_name_is_equal(const h2o_header_t *x, const h2o_header_t *y);
 /**
  * searches for a header of given name (fast, by comparing tokens)
  * @param headers header list
@@ -1509,6 +1516,10 @@ enum {
     H2O_SEND_ERROR_KEEP_HEADERS = 0x2
 };
 
+/**
+ * Add a `date:` header to the response
+ */
+void h2o_resp_add_date_header(h2o_req_t *req);
 /**
  * sends the given string as the response
  */
@@ -1963,6 +1974,15 @@ void h2o_http2_debug_state_register_configurator(h2o_globalconf_t *conf);
 extern pthread_mutex_t h2o_conn_id_mutex;
 #endif
 
+inline int h2o_header_name_is_equal(const h2o_header_t *x, const h2o_header_t *y)
+{
+    if (x->name == y->name) {
+        return 1;
+    } else {
+        return h2o_memis(x->name->base, x->name->len, y->name->base, y->name->len);
+    }
+}
+
 inline h2o_conn_t *h2o_create_connection(size_t sz, h2o_context_t *ctx, h2o_hostconf_t **hosts, struct timeval connected_at,
                                          const h2o_conn_callbacks_t *callbacks)
 {
@@ -2115,7 +2135,8 @@ static inline void h2o_context_set_logger_context(h2o_context_t *ctx, h2o_logger
 static inline void h2o_doublebuffer_init(h2o_doublebuffer_t *db, h2o_buffer_prototype_t *prototype)
 {
     h2o_buffer_init(&db->buf, prototype);
-    db->bytes_inflight = 0;
+    db->inflight = 0;
+    db->_bytes_inflight = 0;
 }
 
 static inline void h2o_doublebuffer_dispose(h2o_doublebuffer_t *db)
@@ -2125,7 +2146,8 @@ static inline void h2o_doublebuffer_dispose(h2o_doublebuffer_t *db)
 
 static inline h2o_iovec_t h2o_doublebuffer_prepare(h2o_doublebuffer_t *db, h2o_buffer_t **receiving, size_t max_bytes)
 {
-    assert(db->bytes_inflight == 0);
+    assert(!db->inflight);
+    assert(max_bytes != 0);
 
     if (db->buf->size == 0) {
         if ((*receiving)->size == 0)
@@ -2135,16 +2157,25 @@ static inline h2o_iovec_t h2o_doublebuffer_prepare(h2o_doublebuffer_t *db, h2o_b
         db->buf = *receiving;
         *receiving = t;
     }
-    if ((db->bytes_inflight = db->buf->size) > max_bytes)
-        db->bytes_inflight = max_bytes;
-    return h2o_iovec_init(db->buf->bytes, db->bytes_inflight);
+    if ((db->_bytes_inflight = db->buf->size) > max_bytes)
+        db->_bytes_inflight = max_bytes;
+    db->inflight = 1;
+    return h2o_iovec_init(db->buf->bytes, db->_bytes_inflight);
+}
+
+static inline void h2o_doublebuffer_prepare_empty(h2o_doublebuffer_t *db)
+{
+    assert(!db->inflight);
+    db->inflight = 1;
 }
 
 static inline void h2o_doublebuffer_consume(h2o_doublebuffer_t *db)
 {
-    assert(db->bytes_inflight != 0);
-    h2o_buffer_consume(&db->buf, db->bytes_inflight);
-    db->bytes_inflight = 0;
+    assert(db->inflight);
+    db->inflight = 0;
+
+    h2o_buffer_consume(&db->buf, db->_bytes_inflight);
+    db->_bytes_inflight = 0;
 }
 
 #define COMPUTE_DURATION(name, from, until)                                                                                        \

diff --git a/include/h2o/http2_internal.h b/include/h2o/http2_internal.h
@@ -69,8 +69,7 @@ size_t h2o_hpack_encode_string(uint8_t *dst, const char *s, size_t len);
 void h2o_hpack_flatten_request(h2o_buffer_t **buf, h2o_hpack_header_table_t *header_table, uint32_t stream_id,
                                size_t max_frame_size, h2o_req_t *req, uint32_t parent_stream_id);
 void h2o_hpack_flatten_response(h2o_buffer_t **buf, h2o_hpack_header_table_t *header_table, uint32_t stream_id,
-                                size_t max_frame_size, h2o_res_t *res, h2o_timestamp_t *ts, const h2o_iovec_t *server_name,
-                                size_t content_length);
+                                size_t max_frame_size, h2o_res_t *res, const h2o_iovec_t *server_name, size_t content_length);
 static h2o_hpack_header_table_entry_t *h2o_hpack_header_table_get(h2o_hpack_header_table_t *table, size_t index);
 
 /* frames */

diff --git a/include/h2o/mruby_.h b/include/h2o/mruby_.h
@@ -52,8 +52,6 @@ enum {
     H2O_MRUBY_LIT_RACK_ERRORS,
     H2O_MRUBY_LIT_SERVER_SOFTWARE,
     H2O_MRUBY_LIT_SERVER_SOFTWARE_VALUE,
-    H2O_MRUBY_LIT_SEPARATOR_COMMA,
-    H2O_MRUBY_LIT_SEPARATOR_SEMICOLON,
     H2O_MRUBY_PROC_EACH_TO_ARRAY,
     H2O_MRUBY_PROC_APP_TO_FIBER,
 
@@ -128,8 +126,12 @@ typedef struct st_h2o_mruby_generator_t {
 } h2o_mruby_generator_t;
 
 #define h2o_mruby_assert(mrb)                                                                                                      \
-    if (mrb->exc != NULL)                                                                                                          \
-    h2o_mruby__assert_failed(mrb, __FILE__, __LINE__)
+    do {                                                                                                                           \
+        if (mrb->exc != NULL)                                                                                                      \
+            h2o_mruby__abort_exc(mrb, "unexpected ruby error", __FILE__, __LINE__);                                                \
+    } while (0)
+
+#define h2o_mruby_new_str(mrb, s, l) h2o_mruby__new_str((mrb), (s), (l), __FILE__, __LINE__)
 
 /* source files using this macro should include mruby/throw.h */
 #define H2O_MRUBY_EXEC_GUARD(block)                                                                                                \
@@ -152,7 +154,8 @@ typedef struct st_h2o_mruby_generator_t {
     } while (0)
 
 /* handler/mruby.c */
-void h2o_mruby__assert_failed(mrb_state *mrb, const char *file, int line);
+void h2o_mruby__abort_exc(mrb_state *mrb, const char *mess, const char *file, int line);
+mrb_value h2o_mruby__new_str(mrb_state *mrb, const char *s, size_t len, const char *file, int line);
 mrb_value h2o_mruby_to_str(mrb_state *mrb, mrb_value v);
 mrb_value h2o_mruby_eval_expr(mrb_state *mrb, const char *expr);
 void h2o_mruby_define_callback(mrb_state *mrb, const char *name, h2o_mruby_callback_t callback);

diff --git a/lib/core/config.c b/lib/core/config.c
@@ -186,6 +186,7 @@ void h2o_config_init(h2o_globalconf_t *config)
     config->proxy.first_byte_timeout = H2O_DEFAULT_PROXY_IO_TIMEOUT;
     config->proxy.emit_x_forwarded_headers = 1;
     config->proxy.emit_via_header = 1;
+    config->proxy.emit_missing_date_header = 1;
     config->http2.max_concurrent_requests_per_connection = H2O_HTTP2_SETTINGS_HOST.max_concurrent_streams;
     config->http2.max_streams_for_priority = 16;
     config->http2.latency_optimization.min_rtt = 50; // milliseconds

diff --git a/lib/core/configurator.c b/lib/core/configurator.c
@@ -320,6 +320,10 @@ static int on_config_paths(h2o_configurator_command_t *cmd, h2o_configurator_con
             h2o_configurator_errprintf(cmd, key, "key (representing the virtual path) must be a string");
             return -1;
         }
+        if (strlen(key->data.scalar) == 0) {
+            h2o_configurator_errprintf(cmd, key, "key (representing the virtual path) must not be an empty string");
+            return -1;
+        }
     }
     qsort(node->data.mapping.elements, node->data.mapping.size, sizeof(node->data.mapping.elements[0]),
           (int (*)(const void *, const void *))sort_from_longer_paths);

diff --git a/lib/core/proxy.c b/lib/core/proxy.c
@@ -336,8 +336,6 @@ static void do_send(struct rp_generator_t *self)
     size_t veccnt;
     h2o_send_state_t ststate;
 
-    assert(self->sending.bytes_inflight == 0);
-
     vecs[0] = h2o_doublebuffer_prepare(&self->sending,
                                        self->client != NULL ? &self->client->sock->input : &self->last_content_before_send,
                                        self->src_req->preferred_chunk_size);
@@ -416,7 +414,7 @@ static int on_body(h2o_http1client_t *client, const char *errstr)
             self->had_body_error = 1;
         }
     }
-    if (self->sending.bytes_inflight == 0)
+    if (!self->sending.inflight)
         do_send(self);
 
     if (self->client && self->client->sock && overrides && self->client->sock->input->size > overrides->max_buffer_size) {
@@ -444,6 +442,8 @@ static h2o_http1client_body_cb on_head(h2o_http1client_t *client, const char *er
     struct rp_generator_t *self = client->data;
     h2o_req_t *req = self->src_req;
     size_t i;
+    int emit_missing_date_header = req->conn->ctx->globalconf->proxy.emit_missing_date_header;
+    int seen_date_header = 0;
 
     if (errstr != NULL && errstr != h2o_http1client_error_is_eos) {
         self->client = NULL;
@@ -499,6 +499,8 @@ static h2o_http1client_body_cb on_head(h2o_http1client_t *client, const char *er
             } else if (token == H2O_TOKEN_X_COMPRESS_HINT) {
                 req->compress_hint = compress_hint_to_enum(headers[i].value.base, headers[i].value.len);
                 goto Skip;
+            } else if (token == H2O_TOKEN_DATE) {
+                seen_date_header = 1;
             }
         /* default behaviour, transfer the header downstream */
         AddHeaderDuped:
@@ -514,6 +516,9 @@ static h2o_http1client_body_cb on_head(h2o_http1client_t *client, const char *er
         }
     }
 
+    if (!seen_date_header && emit_missing_date_header)
+        h2o_resp_add_date_header(req);
+
     if (self->is_websocket_handshake && req->res.status == 101) {
         h2o_http1client_ctx_t *client_ctx = get_client_ctx(req);
         assert(client_ctx->websocket_timeout != NULL);
@@ -531,6 +536,16 @@ static h2o_http1client_body_cb on_head(h2o_http1client_t *client, const char *er
         return NULL;
     }
 
+    /* We currently fail to notify the protocol handler that the headers are complete (by invoking h2o_send(NULL, 0)) if the body
+     * received from upstream is using chunked encoding and if only an incomplete chunk header (i.e. chunk-size CR LF CR LF) was
+     * received along with the HTTP headers. However it is not a big deal; we are only failing to "optimize" for a theoretical
+     * corner case.
+     */
+    if (self->client->sock->input->size == rlen) {
+        h2o_doublebuffer_prepare_empty(&self->sending);
+        h2o_send(req, NULL, 0, H2O_SEND_STATE_IN_PROGRESS);
+    }
+
     return on_body;
 }
 

diff --git a/lib/core/request.c b/lib/core/request.c
@@ -710,3 +710,10 @@ h2o_iovec_t h2o_push_path_in_link_header(h2o_req_t *req, const char *value, size
 
     return ret;
 }
+
+void h2o_resp_add_date_header(h2o_req_t *req)
+{
+    h2o_timestamp_t ts;
+    h2o_get_timestamp(req->conn->ctx, &req->pool, &ts);
+    h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_DATE, NULL, ts.str->rfc1123, strlen(ts.str->rfc1123));
+}
diff --git a/lib/core/token_table.h b/lib/core/token_table.h
@@ -46,7 +46,7 @@ h2o_token_t h2o__tokens[] = {{{H2O_STRLIT(":authority")}, 1, 0, 0, 0, 0, 0, 0},
                              {{H2O_STRLIT("content-range")}, 30, 0, 0, 0, 0, 0, 0},
                              {{H2O_STRLIT("content-type")}, 31, 0, 0, 0, 0, 0, 0},
                              {{H2O_STRLIT("cookie")}, 32, 0, 0, 0, 0, 0, 1},
-                             {{H2O_STRLIT("date")}, 33, 0, 1, 0, 0, 0, 0},
+                             {{H2O_STRLIT("date")}, 33, 0, 0, 0, 0, 0, 0},
                              {{H2O_STRLIT("etag")}, 34, 0, 0, 0, 0, 0, 0},
                              {{H2O_STRLIT("expect")}, 35, 0, 0, 1, 0, 0, 0},
                              {{H2O_STRLIT("expires")}, 36, 0, 0, 0, 0, 0, 0},