sul: all timed objects use a single pt sul list

wsi timeout, wsi hrtimer, sequencer timeout and vh-protocol timer
all now participate on a single sorted us list.

The whole idea of polling wakes is thrown out, poll waits ignore the
timeout field and always use infinite timeouts.

Introduce a public api that can schedule its own callback from the event
loop with us resolution (usually ms is all the platform can do).

Upgrade timeouts and sequencer timeouts to also be able to use us resolution.

Introduce a prepared fakewsi in the pt, so we don't have to allocate
one on the heap when we need it.

Directly handle vh-protocol timer if LWS_MAX_SMP == 1

CMakeLists.txt

@@ -219,6 +219,7 @@ if (NOT LWS_WITH_NETWORK)
 	set(LWS_WITH_HTTP_BROTLI 0)
 	set(LWS_WITH_POLL 0)
 	set(LWS_WITH_SEQUENCER 0)
+	set(LWS_ROLE_DBUS 0)
 endif()
 
 if (LWS_WITH_STRUCT_SQLITE3)
@@ -1281,6 +1282,7 @@ else()
 				list(APPEND SOURCES lib/plat/esp32/esp32-helpers.c)
 			endif()
 		else()
+			set(LWS_PLAT_UNIX 1)
 			list(APPEND SOURCES
 				lib/plat/unix/unix-caps.c
 				lib/plat/unix/unix-file.c
@@ -1322,11 +1324,6 @@ if (LWS_WITH_HTTP_PROXY)
 		lib/roles/http/server/rewrite.c)
 endif()
 
-if (LWS_WITH_NETWORK)
-	list(APPEND SOURCES
-		lib/event-libs/common.c)
-endif()
-
 if (LWS_WITH_POLL AND LWS_WITH_NETWORK)
 	list(APPEND SOURCES
 		lib/event-libs/poll/poll.c)

cmake/lws_config.h.in

@@ -80,6 +80,7 @@
 #cmakedefine LWS_OPENSSL_CLIENT_CERTS "${LWS_OPENSSL_CLIENT_CERTS}"
 #cmakedefine LWS_OPENSSL_SUPPORT
 #cmakedefine LWS_PLAT_OPTEE
+#cmakedefine LWS_PLAT_UNIX
 #cmakedefine LWS_ROLE_CGI
 #cmakedefine LWS_ROLE_DBUS
 #cmakedefine LWS_ROLE_H1

include/libwebsockets.h

@@ -522,13 +522,13 @@ struct lws;
 #include <libwebsockets/lws-http.h>
 #include <libwebsockets/lws-spa.h>
 #include <libwebsockets/lws-purify.h>
+#include <libwebsockets/lws-misc.h>
 #include <libwebsockets/lws-timeout-timer.h>
 #include <libwebsockets/lws-service.h>
 #include <libwebsockets/lws-write.h>
 #include <libwebsockets/lws-writeable.h>
 #include <libwebsockets/lws-adopt.h>
 #include <libwebsockets/lws-network-helper.h>
-#include <libwebsockets/lws-misc.h>
 #include <libwebsockets/lws-ring.h>
 #include <libwebsockets/lws-sha1-base64.h>
 #include <libwebsockets/lws-x509.h>

include/libwebsockets/lws-service.h

@@ -33,9 +33,7 @@
 /**
  * lws_service() - Service any pending websocket activity
  * \param context:	Websocket context
- * \param timeout_ms:	Timeout for poll; 0 means return immediately if nothing needed
- *		service otherwise block and service immediately, returning
- *		after the timeout if nothing needed service.
+ * \param timeout_ms:	Set to 0; ignored; for backward compatibility
  *
  *	This function deals with any pending websocket traffic, for three
  *	kinds of event.  It handles these events on both server and client
@@ -46,20 +44,8 @@
  *	2) Call the receive callback for incoming frame data received by
  *	    server or client connections.
  *
- *	You need to call this service function periodically to all the above
- *	functions to happen; if your application is single-threaded you can
- *	just call it in your main event loop.
- *
- *	Alternatively you can fork a new process that asynchronously handles
- *	calling this service in a loop.  In that case you are happy if this
- *	call blocks your thread until it needs to take care of something and
- *	would call it with a large nonzero timeout.  Your loop then takes no
- *	CPU while there is nothing happening.
- *
- *	If you are calling it in a single-threaded app, you don't want it to
- *	wait around blocking other things in your loop from happening, so you
- *	would call it with a timeout_ms of 0, so it returns immediately if
- *	nothing is pending, or as soon as it services whatever was pending.
+ *  Since v4.0 internally the timeout wait is ignored, the lws scheduler is
+ *  smart enough to stay asleep until an event is queued.
  */
 LWS_VISIBLE LWS_EXTERN int
 lws_service(struct lws_context *context, int timeout_ms);
@@ -68,9 +54,7 @@ lws_service(struct lws_context *context, int timeout_ms);
  * lws_service_tsi() - Service any pending websocket activity
  *
  * \param context:	Websocket context
- * \param timeout_ms:	Timeout for poll; 0 means return immediately if nothing needed
- *		service otherwise block and service immediately, returning
- *		after the timeout if nothing needed service.
+ * \param timeout_ms:	Set to 0; ignored; for backwards compatibility
  * \param tsi:		Thread service index, starting at 0
  *
  * Same as lws_service(), but for a specific thread service index.  Only needed
@@ -126,10 +110,6 @@ lws_cancel_service(struct lws_context *context);
  * If the socket is foreign to lws, it leaves revents alone.  So you can
  * see if you should service yourself by checking the pollfd revents
  * after letting lws try to service it.
- *
- * You should also call this with pollfd = NULL to just allow the
- * once-per-second global timeout checks; if less than a second since the last
- * check it returns immediately then.
  */
 LWS_VISIBLE LWS_EXTERN int
 lws_service_fd(struct lws_context *context, struct lws_pollfd *pollfd);

include/libwebsockets/lws-timeout-timer.h

@@ -150,19 +150,18 @@ lws_set_timer_usecs(struct lws *wsi, lws_usec_t usecs);
 
 /*
  * lws_timed_callback_vh_protocol() - calls back a protocol on a vhost after
- * 					the specified delay
+ * 					the specified delay in seconds
  *
  * \param vh:	 the vhost to call back
  * \param protocol: the protocol to call back
  * \param reason: callback reason
- * \param secs:	how many seconds in the future to do the callback.  Set to
- *		-1 to cancel the timer callback.
+ * \param secs:	how many seconds in the future to do the callback.
  *
  * Callback the specified protocol with a fake wsi pointing to the specified
  * vhost and protocol, with the specified reason, at the specified time in the
  * future.
  *
- * Returns 0 if OK.
+ * Returns 0 if OK or 1 on OOM.
  *
  * In the multithreaded service case, the callback will occur in the same
  * service thread context as the call to this api that requested it.  If it is
@@ -172,4 +171,64 @@ LWS_VISIBLE LWS_EXTERN int
 lws_timed_callback_vh_protocol(struct lws_vhost *vh,
 			       const struct lws_protocols *prot,
 			       int reason, int secs);
+
+/*
+ * lws_timed_callback_vh_protocol_us() - calls back a protocol on a vhost after
+ * 					 the specified delay in us
+ *
+ * \param vh:	 the vhost to call back
+ * \param protocol: the protocol to call back
+ * \param reason: callback reason
+ * \param us:	how many us in the future to do the callback.
+ *
+ * Callback the specified protocol with a fake wsi pointing to the specified
+ * vhost and protocol, with the specified reason, at the specified time in the
+ * future.
+ *
+ * Returns 0 if OK or 1 on OOM.
+ *
+ * In the multithreaded service case, the callback will occur in the same
+ * service thread context as the call to this api that requested it.  If it is
+ * called from a non-service thread, tsi 0 will handle it.
+ */
+LWS_VISIBLE LWS_EXTERN int
+lws_timed_callback_vh_protocol_us(struct lws_vhost *vh,
+				  const struct lws_protocols *prot, int reason,
+				  lws_usec_t us);
+
+
+typedef struct lws_sorted_usec_list lws_sorted_usec_list_t;
+typedef void (*sul_cb_t)(lws_sorted_usec_list_t *sul);
+
+typedef struct lws_sorted_usec_list {
+	struct lws_dll2 list;	/* simplify the code by keeping this at start */
+	sul_cb_t	cb;
+	lws_usec_t	us;
+} lws_sorted_usec_list_t;
+
+
+/*
+ * lws_sul_schedule() - schedule a callback
+ *
+ * \param context: the lws_context
+ * \param tsi: the thread service index (usually 0)
+ * \param sul: pointer to the sul element
+ * \param cb: the scheduled callback
+ * \param us: the delay before the callback arrives, or
+ *		LWS_SET_TIMER_USEC_CANCEL to cancel it.
+ *
+ * Generic callback-at-a-later time function.  The callback happens on the
+ * event loop thread context.
+ *
+ * Although the api has us resultion, the actual resolution depends on the
+ * platform and is commonly 1ms.
+ *
+ * This doesn't allocate and doesn't fail.
+ *
+ * You can call it again with another us value to change the delay.
+ */
+LWS_VISIBLE LWS_EXTERN void
+lws_sul_schedule(struct lws_context *context, int tsi,
+	         lws_sorted_usec_list_t *sul, sul_cb_t cb, lws_usec_t us);
+
 ///@}

lib/core-net/close.c

@@ -70,7 +70,7 @@ __lws_free_wsi(struct lws *wsi)
 #if defined(LWS_WITH_OPENSSL)
 	__lws_ssl_remove_wsi_from_buffered_list(wsi);
 #endif
-	__lws_remove_from_timeout_list(wsi);
+	__lws_wsi_remove_from_sul(wsi);
 
 	if (wsi->context->event_loop_ops->destroy_wsi)
 		wsi->context->event_loop_ops->destroy_wsi(wsi);
@@ -435,12 +435,13 @@ __lws_close_free_wsi(struct lws *wsi, enum lws_close_status reason,
 	 * delete socket from the internal poll list if still present
 	 */
 	__lws_ssl_remove_wsi_from_buffered_list(wsi);
-	__lws_remove_from_timeout_list(wsi);
-	lws_dll2_remove(&wsi->sul_hrtimer.list);
+	__lws_wsi_remove_from_sul(wsi);
 
 	//if (wsi->told_event_loop_closed) // cgi std close case (dummy-callback)
 	//	return;
 
+	// lwsl_notice("%s: wsi %p, fd %d\n", __func__, wsi, wsi->desc.sockfd);
+
 	/* checking return redundant since we anyway close */
 	if (wsi->desc.sockfd != LWS_SOCK_INVALID)
 		__remove_wsi_socket_from_fds(wsi);

lib/core-net/pollfd.c

@@ -231,13 +231,34 @@ lws_accept_modulation(struct lws_context *context,
 }
 #endif
 
+#if defined(_DEBUG)
+void
+__dump_fds(struct lws_context_per_thread *pt, const char *s)
+{
+	unsigned int n;
+
+	lwsl_warn("%s: fds_count %u, %s\n", __func__, pt->fds_count, s);
+
+	for (n = 0; n < pt->fds_count; n++) {
+		struct lws *wsi = wsi_from_fd(pt->context, pt->fds[n].fd);
+
+		lwsl_warn("  %d: fd %d, wsi %p, pos_in_fds: %d\n",
+			n + 1, pt->fds[n].fd, wsi,
+			wsi ? wsi->position_in_fds_table : -1);
+	}
+}
+#else
+#define __dump_fds(x, y)
+#endif
+
 int
 __insert_wsi_socket_into_fds(struct lws_context *context, struct lws *wsi)
 {
 	struct lws_pollargs pa = { wsi->desc.sockfd, LWS_POLLIN, 0 };
 	struct lws_context_per_thread *pt = &context->pt[(int)wsi->tsi];
 	int ret = 0;
 
+//	__dump_fds(pt, "pre insert");
 
 	lwsl_debug("%s: %p: tsi=%d, sock=%d, pos-in-fds=%d\n",
 		  __func__, wsi, wsi->tsi, wsi->desc.sockfd, pt->fds_count);
@@ -294,6 +315,8 @@ __insert_wsi_socket_into_fds(struct lws_context *context, struct lws *wsi)
 					   wsi->user_space, (void *)&pa, 1))
 		ret = -1;
 
+//	__dump_fds(pt, "post insert");
+
 	return ret;
 }
 
@@ -306,6 +329,8 @@ __remove_wsi_socket_from_fds(struct lws *wsi)
 	struct lws *end_wsi;
 	int v, m, ret = 0;
 
+//	__dump_fds(pt, "pre remove");
+
 #if !defined(_WIN32)
 	if (!wsi->context->max_fds_unrelated_to_ulimit &&
 	    wsi->desc.sockfd - lws_plat_socket_offset() > context->max_fds) {
@@ -334,9 +359,8 @@ __remove_wsi_socket_from_fds(struct lws *wsi)
 		context->event_loop_ops->io(wsi,
 				  LWS_EV_STOP | LWS_EV_READ | LWS_EV_WRITE |
 				  LWS_EV_PREPARE_DELETION);
-
-	/*
-	lwsl_debug("%s: wsi=%p, skt=%d, fds pos=%d, end guy pos=%d, endfd=%d\n",
+/*
+	lwsl_notice("%s: wsi=%p, skt=%d, fds pos=%d, end guy pos=%d, endfd=%d\n",
 		  __func__, wsi, wsi->desc.sockfd, wsi->position_in_fds_table,
 		  pt->fds_count, pt->fds[pt->fds_count - 1].fd); */
 
@@ -395,6 +419,8 @@ __remove_wsi_socket_from_fds(struct lws *wsi)
 					      wsi->user_space, (void *) &pa, 1))
 		ret = -1;
 
+//	__dump_fds(pt, "post remove");
+
 	return ret;
 }